From ce38bf54f6d2b8367859b7c09a01faacd3e53cf2 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 25 Aug 2023 11:39:15 +0900 Subject: [PATCH 001/104] [WIP] simplify gp implementation --- optuna_dashboard/preferential/samplers/_gp.py | 485 ++++++++---------- 1 file changed, 214 insertions(+), 271 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/_gp.py b/optuna_dashboard/preferential/samplers/_gp.py index ff4001c1..e7e6085d 100644 --- a/optuna_dashboard/preferential/samplers/_gp.py +++ b/optuna_dashboard/preferential/samplers/_gp.py @@ -5,19 +5,16 @@ from math import erfc from typing import Any from botorch.acquisition.analytic import LogExpectedImprovement -from botorch.models.gpytorch import GPyTorchModel from botorch.optim import optimize_acqf +import botorch.models.model +import botorch.posteriors.gpytorch + +import botorch import gpytorch.constraints import gpytorch.kernels import gpytorch.likelihoods.gaussian_likelihood -from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood -from gpytorch.likelihoods.gaussian_likelihood import Interval from gpytorch.likelihoods.gaussian_likelihood import Prior -from gpytorch.models.exact_gp import ExactGP import gpytorch.module -from linear_operator.operators import DiagLinearOperator -from linear_operator.operators import LinearOperator -from linear_operator.utils.errors import NotPSDError import numpy as np import optuna from optuna import distributions @@ -26,8 +23,6 @@ from optuna._transform import _SearchSpaceTransform from optuna.distributions import BaseDistribution from optuna.search_space import IntersectionSearchSpace from optuna.trial import FrozenTrial -import pyro -import pyro.infer.autoguide import pyro.infer.mcmc from scipy.special import erfcinv import torch @@ -36,288 +31,252 @@ from torch import Tensor from .._system_attrs import get_preferences -class _WeightedGaussianLikelihood(GaussianLikelihood): - def __init__( - self, - weights: torch.Tensor | None = None, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, - batch_shape: torch.Size = torch.Size(), - **kwargs: Any, - ) -> None: - super().__init__( - noise_prior=noise_prior, - noise_constraint=noise_constraint, - batch_shape=batch_shape, - **kwargs, - ) - self.weights = weights - - def _shaped_noise_covar( - self, base_shape: torch.Size, *params: Any, **kwargs: Any - ) -> Tensor | LinearOperator: - assert self.weights is not None - assert base_shape[-1] == self.weights.shape[-1] - return DiagLinearOperator(1.0 / self.weights) * super()._shaped_noise_covar( - base_shape, *params, **kwargs - ) - - -def _sample_y( - preferences: np.ndarray, - cov_X_X: np.ndarray, - obs_noise_var: float, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: - # TODO: Refactor and write tests for this function. - - N = cov_X_X.shape[0] - M = len(preferences) - cov_X_X = cov_X_X + np.eye(N) * 1e-6 # Add jitter - cov_X_X_chol = np.linalg.cholesky(cov_X_X) - cov_X_X_inv = np.linalg.inv(cov_X_X) - - # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T - - schur = cov_X_X_inv.copy() - np.add.at(schur, (preferences[:, 0], preferences[:, 0]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 1]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 0], preferences[:, 1]), -1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 0]), -1.0 / (2 * obs_noise_var)) - idx_M = np.arange(M) - - schur_inv = np.linalg.inv(schur) - - cov_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] - cov_diff_inv = cov_diff_inv[preferences[:, 0], :] - cov_diff_inv[preferences[:, 1], :] - cov_diff_inv *= -1 / (2 * obs_noise_var) ** 2 - cov_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) - - diffs = _orthants_MVN_Gibbs_sampling( - cov_diff_inv, - cycles=cycles, - initial_sample=initial_sample[:, 0] - initial_sample[:, 1], - rng=rng, - )[-1] - - random_ys = (cov_X_X_chol @ rng.randn(N))[preferences] + np.sqrt(obs_noise_var) * rng.randn( - M, 2 - ) - errors = diffs - (random_ys[:, 0] - random_ys[:, 1]) - cov_diff_inv_errors = cov_diff_inv @ errors - - AT_cov_diff_inv_errors = np.zeros((N,)) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 0], cov_diff_inv_errors) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 1], -cov_diff_inv_errors) - - return ( - random_ys - + (cov_X_X @ AT_cov_diff_inv_errors)[preferences] - + obs_noise_var * np.array([[1, -1]]) * cov_diff_inv_errors[:, None] - ) - - _SQRT2 = math.sqrt(2) - def _orthants_MVN_Gibbs_sampling( - cov_inv: np.ndarray, + cov_inv: torch.Tensor, cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: + initial_sample: torch.Tensor, +) -> torch.Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) - - if initial_sample is None: - sample_chain = np.zeros(dim) - else: + with torch.no_grad(): sample_chain = initial_sample + conditional_std = 1 / torch.sqrt(torch.diag(cov_inv)) + scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - conditional_std = 1 / np.sqrt(np.diag(cov_inv)) + out = torch.empty((cycles + 1, dim), dtype=torch.float64) + out[0, :] = sample_chain - scaled_cov_inv = cov_inv / np.c_[np.diag(cov_inv)] - - out = np.empty((cycles + 1, dim)) - out[0, :] = sample_chain - - for i in range(cycles): - for j in range(dim): - conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain - sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=-conditional_mean / conditional_std[j], rng=rng + for i in range(cycles): + for j in range(dim): + conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain + sample_chain[j] = ( + _one_side_trunc_norm_sampling(lower=float(-conditional_mean / conditional_std[j])) + * conditional_std[j] + + conditional_mean ) - * conditional_std[j] - + conditional_mean - ) - out[i + 1, :] = sample_chain + out[i + 1, :] = sample_chain - return out + return out -def _one_side_trunc_norm_sampling(lower: float, rng: np.random.RandomState) -> float: - return erfcinv(rng.rand() * erfc(lower / _SQRT2)) * _SQRT2 +def _one_side_trunc_norm_sampling(lower: float) -> float: + return erfcinv(torch.rand() * erfc(lower / _SQRT2)) * _SQRT2 +def _compute_cov_diff_diff_inv( + preferences: torch.Tensor, + cov_x_x: torch.Tensor, + obs_noise_var: float, +) -> torch.Tensor: + N = cov_x_x.shape[0] + M = preferences.shape[0] -class _PreferentialGP(GPyTorchModel, ExactGP): - _num_outputs = 1 + # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T + # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) + I_plus_sinv_AT_A_K = torch.eye(N) + A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K, alpha=1 / obs_noise_var) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K, alpha=-1/obs_noise_var) + schur_inv: torch.Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) + cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] + cov_diff_diff_inv = cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + cov_diff_diff_inv *= -1 / (2 * obs_noise_var) ** 2 + idx_M = torch.arange(M) + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) + + return cov_diff_diff_inv + +def _multinormal_logpdf(Sigma_inv: torch.Tensor, x: torch.Tensor): + return -0.5 * x @ Sigma_inv @ x + 0.5 * torch.logdet(Sigma_inv) - 0.5 * x.shape[0] * math.log(2 * math.pi) + +class _SampledGP(botorch.models.model.Model): def __init__( self, kernel: gpytorch.kernels.Kernel, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, + x: torch.Tensor, + preferences: torch.Tensor, + obs_noise_var: torch.Tensor, + diff: torch.Tensor, ) -> None: - GPyTorchModel.__init__(self) - likelihood = _WeightedGaussianLikelihood( - noise_prior=noise_prior, noise_constraint=noise_constraint + self.kernel = kernel + self.x = x + self.preferences = preferences + self.diff = diff + self.obs_noise_var = obs_noise_var + self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel(x).to_dense(), + obs_noise_var=float(obs_noise_var), ) - ExactGP.__init__(self, train_inputs=None, train_targets=None, likelihood=likelihood) - self.covar_module = kernel - self._last_params: dict[str, torch.Tensor] | None = None - self._last_mcmc_step_size: float | None = None + def posterior( + self, + x2: Tensor, + output_indices: list[int] | None = None, + observation_noise: bool = False, + posterior_transform: Any | None = None, + **kwargs: Any, + ) -> botorch.posteriors.gpytorch.GPyTorchPosterior: + assert posterior_transform is None + assert output_indices is None + assert self.x.shape[-1] == x2.shape[-1] - def _pyro_model(self, train_x: torch.Tensor, train_y: torch.Tensor) -> None: - # with gpytorch.settings.fast_computations(False, False, False): - sampled_model = self.pyro_sample_from_prior() + x_expanded = self.x.expand(x2.shape[:-2] + (self.x.shape[-2], x2.shape[-1])) - ys = sampled_model.likelihood(sampled_model.forward(train_x)) + cov_x2_x: torch.Tensor = self.kernel(x2, x_expanded).to_dense() + cov_x2_diff: torch.Tensor = cov_x2_x[..., self.preferences[:, 0]] - cov_x2_x[..., self.preferences[:, 1]] - pyro.sample("y", ys, obs=train_y) + mean: torch.Tensor = cov_x2_diff @ (self._cov_diff_diff_inv @ self.diff) + cov: torch.Tensor = self.kernel(x2).to_dense() - cov_x2_diff @ self._cov_diff_diff_inv @ cov_x2_diff.transpose(-1, -2) + if observation_noise: + idx = torch.arange(cov.shape[-1]) + cov[..., idx, idx] += self.obs_noise_var - def fit_mcmc( - self, X: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState + return botorch.posteriors.gpytorch.GPyTorchPosterior( + distribution=gpytorch.distributions.MultivariateNormal( + mean=mean, + covariance_matrix=cov, + ) + ) + + @property + def batch_shape(self) -> torch.Size: + return torch.Size() + + @property + def num_outputs(self) -> int: + return 1 + + +class _PreferentialGP: + def _kernel_factory(self, lengthscale: torch.Tensor) -> gpytorch.kernels.Kernel: + kernel = gpytorch.kernels.MaternKernel( + nu=2.5, + ard_num_dims=lengthscale.shape[0], + ) + kernel.lengthscale = lengthscale + return kernel + + def _potential_func( + self, + x: torch.Tensor, + preferences: torch.Tensor, + diff: torch.Tensor, + log_lengthscale: torch.Tensor, + log_noise: torch.Tensor, + ) -> torch.Tensor: + + lengthscale = torch.exp(log_lengthscale) + noise = torch.exp(log_noise) + log_transform_jacobian = torch.sum(log_lengthscale) + log_noise + log_prior = self.lengthscale_prior.log_prob(lengthscale) + self.noise_prior.log_prob(noise) + cov_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self._kernel_factory(lengthscale)(x).to_dense(), + obs_noise_var=noise, + ) + log_likelihood = _multinormal_logpdf(cov_inv, diff) + return log_prior + log_likelihood + log_transform_jacobian + + def __init__( + self, + lengthscale_prior: Prior, + noise_prior: Prior, + dims: int, ) -> None: + self.lengthscale_prior: Prior = lengthscale_prior.expand((dims,)) + self.noise_prior = noise_prior + self.dims = dims + + self._x = torch.empty((0, dims), dtype=torch.float64) + self._preferences = torch.empty((0, 2), dtype=torch.float64) + self._diff = torch.empty((0,), dtype=torch.float64) + + initial_raw_params = { + "log_lengthscale": torch.log(self.lengthscale_prior.sample()), + "log_noise": torch.log(self.noise_prior.sample()), + } + + self._potential_func_jit = torch.jit.trace( + self._potential_func, + (self._x, self._preferences, self._diff, initial_raw_params["log_lengthscale"], initial_raw_params["log_noise"]), + ) + + # HMC-Gibbs workarounds + # https://github.com/pyro-ppl/pyro/issues/1926 + + self._nuts = pyro.infer.mcmc.NUTS(potential_fn=lambda z:self._potential_func_jit( + x=self._x, + preferences=self._preferences, + diff=self._diff, + log_lengthscale=z["log_lengthscale"], + log_noise=z["log_noise"], + )) + + self._nuts.initial_params = initial_raw_params + self._nuts.setup(warmup_steps=1e15) # Infinite warmup + self._last_params = initial_raw_params + + def sample_gp( + self, x: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState + ) -> _SampledGP: if len(preferences) == 0: - # Skip actual MCMC computation - self.set_train_data( - inputs=torch.empty((0, X.shape[-1])), - targets=torch.empty((0,)), - strict=False, + return _SampledGP( + kernel=self._kernel_factory(self.lengthscale_prior.sample()), + x=x, + preferences=preferences, + obs_noise_var=self.noise_prior.sample(), + diff=torch.empty((0,), dtype=torch.float64), ) - self.likelihood.weights = torch.empty((0,)) else: - dtype = torch.float64 - cnt = torch.bincount(preferences.reshape(-1)) - mask = cnt > 0 - train_x = X[mask] - weights = cnt[mask] + original_diff_size = len(self._diff) + self._diff.resize_(len(preferences)) + self._diff[original_diff_size:] = 0.0 - assert isinstance(self.likelihood, _WeightedGaussianLikelihood) - self.likelihood.weights = weights - - preferences_np = preferences.detach().numpy() - - all_ys_np = np.zeros((len(preferences), 2)) - train_y = torch.zeros( - ( - len( - train_x, - ) - ), - dtype=dtype, - ) - - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, - ) - warmup_steps = max(0, cycles - 2) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - - raw_params = self._last_params or nuts.initial_params - for i in range(cycles): - params = { - name: nuts.transforms[name].inv(value) for name, value in raw_params.items() - } - _set_params(self, params) - self.set_train_data(train_x, train_y, strict=False) - all_ys_np = _sample_y( - preferences=preferences_np, - cov_X_X=self.covar_module(train_x).detach().numpy(), - obs_noise_var=float(self.likelihood.noise_covar.noise), - cycles=10, - initial_sample=all_ys_np, - rng=rng, + for _ in range(cycles): + kernel = self._kernel_factory(torch.exp(self._last_params["log_lengthscale"])) + cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=kernel(x).to_dense(), + obs_noise_var=float(torch.exp(self._last_params["log_noise"])), ) - ys_sum_np = np.zeros((len(X),)) - np.add.at(ys_sum_np, preferences_np.reshape(-1), all_ys_np.reshape(-1)) - ys_sum = torch.from_numpy(ys_sum_np) - train_y[:] = ys_sum[mask] / cnt[mask] - nuts.clear_cache() - try: - raw_params = nuts.sample(raw_params) - except NotPSDError: - nuts.cleanup() - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, - ) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - raw_params = nuts.initial_params - - params = {name: nuts.transforms[name].inv(value) for name, value in raw_params.items()} - self.set_train_data(train_x, train_y, strict=False) - _set_params(self, params) - - self._last_params = raw_params - self._last_mcmc_step_size = nuts.step_size - nuts.cleanup() - - def forward(self, x: torch.Tensor) -> gpytorch.distributions.MultivariateNormal: - mean_module = gpytorch.means.ZeroMean() - return gpytorch.distributions.MultivariateNormal( - mean_module(x), - self.covar_module(x), - ) - - -def _set_params( - module: gpytorch.Module, - params_dict: dict[str, torch.Tensor], - memo: set | None = None, - prefix: str = "", -) -> None: - if memo is None: - memo = set() - if hasattr(module, "_priors"): - for name, (prior, closure, setting_closure) in module._priors.items(): - if prior is not None and prior not in memo: - memo.add(prior) - setting_closure(module, params_dict[prefix + ("." if prefix else "") + name]) - - for mname, module_ in module.named_children(): - submodule_prefix = prefix + ("." if prefix else "") + mname - _set_params(module_, params_dict, memo=memo, prefix=submodule_prefix) + self._diff = _orthants_MVN_Gibbs_sampling( + cov_inv=cov_diff_diff_inv, + cycles=10, + initial_sample=self._diff + )[:-1] + self._nuts.clear_cache() + self._last_raw_params = self._nuts.sample(self._last_raw_params) + + return _SampledGP( + kernel=self._kernel_factory(torch.exp(self._last_params["log_lengthscale"])), + x=x, + preferences=preferences, + obs_noise_var=torch.exp(self._last_params["log_noise"]), + diff=self._diff, + ) class PreferentialGPSampler(optuna.samplers.BaseSampler): def __init__( self, *, - kernel: gpytorch.kernels.Kernel | None = None, + # kernel_factory: typing.Callable[[int], gpytorch.kernels.Kernel] | None = None, + lengthscale_prior: Prior | None = None, noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, - device: torch.device | None = None, + # device: torch.device | None = None, ) -> None: + self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(3.0, 6.0) + self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(1.1, 10.0) + self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler(seed=self._rng.randint(2**32)) + self._rng = np.random.RandomState(seed) self._search_space = IntersectionSearchSpace() - - self.kernel = kernel - self.noise_prior = noise_prior - self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( - seed=self._rng.randint(2**32), - ) - self.device = device or torch.device("cpu") - self._gp: _PreferentialGP | None = None def reseed_rng(self) -> None: @@ -352,15 +311,9 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): ) dims = len(trans.bounds) self._gp = self._gp or _PreferentialGP( - kernel=self.kernel - or gpytorch.kernels.MaternKernel( - nu=2.5, - ard_num_dims=dims, - lengthscale_prior=gpytorch.priors.GammaPrior(3.0, 6.0), - lengthscale_constraint=gpytorch.constraints.Positive(), - ), - noise_prior=self.noise_prior or gpytorch.priors.GammaPrior(1.1, 2.0), - noise_constraint=gpytorch.constraints.Positive(), + lengthscale_prior=self.lengthscale_prior, + noise_prior=self.noise_prior, + dims=dims, ) ids: dict[int, int] = {} @@ -373,23 +326,12 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): ids[t] = len(ids) params.append(trans.transform(trials[t].params)) pref_ids.append((ids[better], ids[worse])) - dtype = torch.float64 - - params_torch = torch.tensor(np.array(params), dtype=dtype, device=self.device) - pref_ids_torch = torch.tensor( - np.array(pref_ids), - dtype=torch.int32, - device=self.device, - ) - self._gp.fit_mcmc(params_torch, pref_ids_torch, cycles=10, rng=self._rng) - self._gp.eval() - scores = self._gp(params_torch).mean - - best_f = torch.max(scores) - + params_torch = torch.tensor(np.array(params), dtype=torch.float64) + pref_ids_torch = torch.tensor(np.array(pref_ids), dtype=torch.int32) + sampled_gp = self._gp.sample_gp(params_torch, pref_ids_torch, cycles=10, rng=self._rng) acqf = LogExpectedImprovement( - model=self._gp, - best_f=best_f, + model=sampled_gp, + best_f=torch.max(sampled_gp.posterior(params_torch[:, None, :]).mean), ) # TODO: Make it possible to apply it on categorical variables @@ -415,3 +357,4 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution ) + From 17415e9014959e086a0d0384a91f1c429ea59016 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 25 Aug 2023 12:16:25 +0900 Subject: [PATCH 002/104] [WIP] simplify gp implementation --- optuna_dashboard/preferential/samplers/_gp.py | 191 +++++++++--------- 1 file changed, 98 insertions(+), 93 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/_gp.py b/optuna_dashboard/preferential/samplers/_gp.py index e7e6085d..d8b6d7bf 100644 --- a/optuna_dashboard/preferential/samplers/_gp.py +++ b/optuna_dashboard/preferential/samplers/_gp.py @@ -1,38 +1,25 @@ from __future__ import annotations -import math from math import erfc +from math import sqrt from typing import Any -from botorch.acquisition.analytic import LogExpectedImprovement -from botorch.optim import optimize_acqf +import botorch.acquisition.analytic import botorch.models.model +import botorch.optim import botorch.posteriors.gpytorch - -import botorch -import gpytorch.constraints import gpytorch.kernels -import gpytorch.likelihoods.gaussian_likelihood from gpytorch.likelihoods.gaussian_likelihood import Prior -import gpytorch.module import numpy as np import optuna -from optuna import distributions -from optuna import Study -from optuna._transform import _SearchSpaceTransform -from optuna.distributions import BaseDistribution -from optuna.search_space import IntersectionSearchSpace -from optuna.trial import FrozenTrial +import optuna._transform import pyro.infer.mcmc from scipy.special import erfcinv import torch -from torch import Tensor from .._system_attrs import get_preferences -_SQRT2 = math.sqrt(2) - def _orthants_MVN_Gibbs_sampling( cov_inv: torch.Tensor, cycles: int, @@ -52,7 +39,9 @@ def _orthants_MVN_Gibbs_sampling( for j in range(dim): conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain sample_chain[j] = ( - _one_side_trunc_norm_sampling(lower=float(-conditional_mean / conditional_std[j])) + _one_side_trunc_norm_sampling( + lower=float(-conditional_mean / conditional_std[j]) + ) * conditional_std[j] + conditional_mean ) @@ -61,9 +50,13 @@ def _orthants_MVN_Gibbs_sampling( return out +_SQRT2 = sqrt(2) + + def _one_side_trunc_norm_sampling(lower: float) -> float: return erfcinv(torch.rand() * erfc(lower / _SQRT2)) * _SQRT2 + def _compute_cov_diff_diff_inv( preferences: torch.Tensor, cov_x_x: torch.Tensor, @@ -78,18 +71,26 @@ def _compute_cov_diff_diff_inv( I_plus_sinv_AT_A_K = torch.eye(N) A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K, alpha=1 / obs_noise_var) - I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K, alpha=-1/obs_noise_var) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K, alpha=-1 / obs_noise_var) schur_inv: torch.Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] - cov_diff_diff_inv = cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + cov_diff_diff_inv = ( + cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + ) cov_diff_diff_inv *= -1 / (2 * obs_noise_var) ** 2 idx_M = torch.arange(M) cov_diff_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) return cov_diff_diff_inv + def _multinormal_logpdf(Sigma_inv: torch.Tensor, x: torch.Tensor): - return -0.5 * x @ Sigma_inv @ x + 0.5 * torch.logdet(Sigma_inv) - 0.5 * x.shape[0] * math.log(2 * math.pi) + return ( + -0.5 * x @ Sigma_inv @ x + + 0.5 * torch.logdet(Sigma_inv) + - 0.5 * x.shape[0] * math.log(2 * math.pi) + ) + class _SampledGP(botorch.models.model.Model): def __init__( @@ -113,7 +114,7 @@ class _SampledGP(botorch.models.model.Model): def posterior( self, - x2: Tensor, + x2: torch.Tensor, output_indices: list[int] | None = None, observation_noise: bool = False, posterior_transform: Any | None = None, @@ -126,17 +127,21 @@ class _SampledGP(botorch.models.model.Model): x_expanded = self.x.expand(x2.shape[:-2] + (self.x.shape[-2], x2.shape[-1])) cov_x2_x: torch.Tensor = self.kernel(x2, x_expanded).to_dense() - cov_x2_diff: torch.Tensor = cov_x2_x[..., self.preferences[:, 0]] - cov_x2_x[..., self.preferences[:, 1]] + cov_x2_diff: torch.Tensor = ( + cov_x2_x[..., self.preferences[:, 0]] - cov_x2_x[..., self.preferences[:, 1]] + ) mean: torch.Tensor = cov_x2_diff @ (self._cov_diff_diff_inv @ self.diff) - cov: torch.Tensor = self.kernel(x2).to_dense() - cov_x2_diff @ self._cov_diff_diff_inv @ cov_x2_diff.transpose(-1, -2) + cov: torch.Tensor = self.kernel( + x2 + ).to_dense() - cov_x2_diff @ self._cov_diff_diff_inv @ cov_x2_diff.transpose(-1, -2) if observation_noise: idx = torch.arange(cov.shape[-1]) cov[..., idx, idx] += self.obs_noise_var return botorch.posteriors.gpytorch.GPyTorchPosterior( distribution=gpytorch.distributions.MultivariateNormal( - mean=mean, + mean=mean, covariance_matrix=cov, ) ) @@ -158,16 +163,15 @@ class _PreferentialGP: ) kernel.lengthscale = lengthscale return kernel - - def _potential_func( - self, - x: torch.Tensor, - preferences: torch.Tensor, - diff: torch.Tensor, - log_lengthscale: torch.Tensor, - log_noise: torch.Tensor, - ) -> torch.Tensor: + def _potential_func( + self, + x: torch.Tensor, + preferences: torch.Tensor, + diff: torch.Tensor, + log_lengthscale: torch.Tensor, + log_noise: torch.Tensor, + ) -> torch.Tensor: lengthscale = torch.exp(log_lengthscale) noise = torch.exp(log_noise) log_transform_jacobian = torch.sum(log_lengthscale) + log_noise @@ -201,27 +205,33 @@ class _PreferentialGP: self._potential_func_jit = torch.jit.trace( self._potential_func, - (self._x, self._preferences, self._diff, initial_raw_params["log_lengthscale"], initial_raw_params["log_noise"]), + ( + self._x, + self._preferences, + self._diff, + initial_raw_params["log_lengthscale"], + initial_raw_params["log_noise"], + ), ) # HMC-Gibbs workarounds # https://github.com/pyro-ppl/pyro/issues/1926 - self._nuts = pyro.infer.mcmc.NUTS(potential_fn=lambda z:self._potential_func_jit( - x=self._x, - preferences=self._preferences, - diff=self._diff, - log_lengthscale=z["log_lengthscale"], - log_noise=z["log_noise"], - )) + self._nuts = pyro.infer.mcmc.NUTS( + potential_fn=lambda z: self._potential_func_jit( + x=self._x, + preferences=self._preferences, + diff=self._diff, + log_lengthscale=z["log_lengthscale"], + log_noise=z["log_noise"], + ) + ) self._nuts.initial_params = initial_raw_params - self._nuts.setup(warmup_steps=1e15) # Infinite warmup + self._nuts.setup(warmup_steps=1e15) # Infinite warmup self._last_params = initial_raw_params - def sample_gp( - self, x: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState - ) -> _SampledGP: + def sample_gp(self, x: torch.Tensor, preferences: torch.Tensor, cycles: int) -> _SampledGP: if len(preferences) == 0: return _SampledGP( kernel=self._kernel_factory(self.lengthscale_prior.sample()), @@ -231,7 +241,6 @@ class _PreferentialGP: diff=torch.empty((0,), dtype=torch.float64), ) else: - original_diff_size = len(self._diff) self._diff.resize_(len(preferences)) self._diff[original_diff_size:] = 0.0 @@ -244,14 +253,12 @@ class _PreferentialGP: obs_noise_var=float(torch.exp(self._last_params["log_noise"])), ) self._diff = _orthants_MVN_Gibbs_sampling( - cov_inv=cov_diff_diff_inv, - cycles=10, - initial_sample=self._diff + cov_inv=cov_diff_diff_inv, cycles=10, initial_sample=self._diff )[:-1] self._nuts.clear_cache() self._last_raw_params = self._nuts.sample(self._last_raw_params) - + return _SampledGP( kernel=self._kernel_factory(torch.exp(self._last_params["log_lengthscale"])), x=x, @@ -260,23 +267,24 @@ class _PreferentialGP: diff=self._diff, ) + class PreferentialGPSampler(optuna.samplers.BaseSampler): def __init__( self, *, - # kernel_factory: typing.Callable[[int], gpytorch.kernels.Kernel] | None = None, lengthscale_prior: Prior | None = None, noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, - # device: torch.device | None = None, ) -> None: self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(3.0, 6.0) self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(1.1, 10.0) - self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler(seed=self._rng.randint(2**32)) + self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( + seed=self._rng.randint(2**32) + ) self._rng = np.random.RandomState(seed) - self._search_space = IntersectionSearchSpace() + self._search_space = optuna.search_space.IntersectionSearchSpace() self._gp: _PreferentialGP | None = None def reseed_rng(self) -> None: @@ -284,58 +292,56 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): self._rng = np.random.RandomState() def infer_relative_search_space( - self, study: Study, trial: FrozenTrial - ) -> dict[str, BaseDistribution]: + self, study: optuna.Study, trial: optuna.trial.FrozenTrial + ) -> dict[str, optuna.distributions.BaseDistribution]: return self._search_space.calculate(study) def sample_relative( self, - study: Study, - trial: FrozenTrial, - search_space: dict[str, BaseDistribution], + study: optuna.Study, + trial: optuna.trial.FrozenTrial, + search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: + preferences = get_preferences(study._study_id, study._storage) + if len(preferences) == 0: + return {} + + trials = study.get_trials(deepcopy=False) + trials_with_preference = list({t for (b, w) in preferences for t in (b, w)}) + ids = {t: i for i, t in enumerate(trials_with_preference)} + + trans = optuna._transform._SearchSpaceTransform( + search_space, transform_log=True, transform_step=True, transform_0_1=True + ) + params = torch.tensor( + [trans.transform(trials[t].params) for t in trials_with_preference], + dtype=torch.float64, + ) + pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) + with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) pyro.set_rng_seed(self._rng.randint(2**32)) - if len(search_space) == 0: - return {} - - preferences = get_preferences(study._study_id, study._storage) - trials = study.get_trials(deepcopy=False) - if len(preferences) == 0: - return {} - - trans = _SearchSpaceTransform( - search_space, transform_log=True, transform_step=True, transform_0_1=True - ) - dims = len(trans.bounds) self._gp = self._gp or _PreferentialGP( lengthscale_prior=self.lengthscale_prior, noise_prior=self.noise_prior, - dims=dims, + dims=len(trans.bounds), ) + if self._gp.dims != len(trans.bounds): + raise NotImplementedError( + "The search space has changed. " + "Dynamic search space is not supported in PreferentialGPSampler." + ) - ids: dict[int, int] = {} - params: list[torch.Tensor] = [] - pref_ids: list[tuple[int, int]] = [] - - for better, worse in preferences: - for t in (better, worse): - if t not in ids: - ids[t] = len(ids) - params.append(trans.transform(trials[t].params)) - pref_ids.append((ids[better], ids[worse])) - params_torch = torch.tensor(np.array(params), dtype=torch.float64) - pref_ids_torch = torch.tensor(np.array(pref_ids), dtype=torch.int32) - sampled_gp = self._gp.sample_gp(params_torch, pref_ids_torch, cycles=10, rng=self._rng) - acqf = LogExpectedImprovement( + sampled_gp = self._gp.sample_gp(params, pref_ids, cycles=10) + acqf = botorch.acquisition.analytic.LogExpectedImprovement( model=sampled_gp, - best_f=torch.max(sampled_gp.posterior(params_torch[:, None, :]).mean), + best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) # TODO: Make it possible to apply it on categorical variables - candidates, _ = optimize_acqf( + candidates, _ = botorch.optim.optimize_acqf( acq_function=acqf, bounds=torch.from_numpy(trans.bounds.T), q=1, @@ -349,12 +355,11 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): def sample_independent( self, - study: Study, - trial: FrozenTrial, + study: optuna.Study, + trial: optuna.trial.FrozenTrial, param_name: str, - param_distribution: distributions.BaseDistribution, + param_distribution: optuna.distributions.BaseDistribution, ) -> Any: return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution ) - From 28062b648a98b51dc7fb3694fd6e898308250790 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Wed, 30 Aug 2023 09:55:47 +0900 Subject: [PATCH 003/104] Simplify GP Implementation --- optuna_dashboard/preferential/samplers/_gp.py | 244 ++++++++++-------- 1 file changed, 135 insertions(+), 109 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/_gp.py b/optuna_dashboard/preferential/samplers/_gp.py index d8b6d7bf..2f8bdba6 100644 --- a/optuna_dashboard/preferential/samplers/_gp.py +++ b/optuna_dashboard/preferential/samplers/_gp.py @@ -1,25 +1,24 @@ +#%% from __future__ import annotations -from math import erfc -from math import sqrt -from typing import Any +import math +from typing import Any, Callable import botorch.acquisition.analytic import botorch.models.model import botorch.optim import botorch.posteriors.gpytorch import gpytorch.kernels +import gpytorch.constraints from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna import optuna._transform import pyro.infer.mcmc -from scipy.special import erfcinv import torch from .._system_attrs import get_preferences - def _orthants_MVN_Gibbs_sampling( cov_inv: torch.Tensor, cycles: int, @@ -27,94 +26,98 @@ def _orthants_MVN_Gibbs_sampling( ) -> torch.Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) - with torch.no_grad(): - sample_chain = initial_sample - conditional_std = 1 / torch.sqrt(torch.diag(cov_inv)) - scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - out = torch.empty((cycles + 1, dim), dtype=torch.float64) - out[0, :] = sample_chain + sample_chain = initial_sample + conditional_std = 1 / torch.sqrt(torch.diag(cov_inv)) + scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - for i in range(cycles): - for j in range(dim): - conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain - sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=float(-conditional_mean / conditional_std[j]) - ) - * conditional_std[j] - + conditional_mean + out = torch.empty((cycles + 1, dim), dtype=torch.float64) + out[0, :] = sample_chain + + for i in range(cycles): + for j in range(dim): + conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain + sample_chain[j] = ( + _one_side_trunc_norm_sampling( + lower=-conditional_mean / conditional_std[j] ) - out[i + 1, :] = sample_chain + * conditional_std[j] + + conditional_mean + ) + out[i + 1, :] = sample_chain - return out + return out + +def _one_side_trunc_norm_sampling(lower: torch.Tensor) -> torch.Tensor: + if lower > 4.0: + r = torch.max(torch.tensor(1e-300), torch.rand(torch.Size(()), dtype=torch.float64)) + return (lower * lower - 2 * r.log()).sqrt() + else: + SQRT2 = math.sqrt(2) + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + while 1 - r == 1: + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + return torch.erfinv(1 - r) * SQRT2 -_SQRT2 = sqrt(2) - - -def _one_side_trunc_norm_sampling(lower: float) -> float: - return erfcinv(torch.rand() * erfc(lower / _SQRT2)) * _SQRT2 - - -def _compute_cov_diff_diff_inv( +_orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) + +def _compute_cov_diff_diff_inv_and_logdet( preferences: torch.Tensor, cov_x_x: torch.Tensor, - obs_noise_var: float, -) -> torch.Tensor: + obs_noise_var: torch.Tensor, +) -> tuple[torch.Tensor, torch.Tensor]: N = cov_x_x.shape[0] M = preferences.shape[0] # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) - I_plus_sinv_AT_A_K = torch.eye(N) + # det(sI + A K A^T) = s^N det(I + s^-1 A K A^T) = s^N det(I + s^-1 A^T A K) + + I_plus_sinv_AT_A_K = torch.eye(N, dtype=torch.float64) A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] - I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K, alpha=1 / obs_noise_var) - I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K, alpha=-1 / obs_noise_var) - schur_inv: torch.Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / obs_noise_var)) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / obs_noise_var)) + lu, piv = torch.linalg.lu_factor(I_plus_sinv_AT_A_K) + + logdet = -(lu.diagonal().abs() * obs_noise_var).log().sum() + + schur_inv: torch.Tensor = torch.linalg.lu_solve(lu, piv, cov_x_x, left=False) cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] cov_diff_diff_inv = ( cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] ) - cov_diff_diff_inv *= -1 / (2 * obs_noise_var) ** 2 + cov_diff_diff_inv *= -1 / obs_noise_var ** 2 idx_M = torch.arange(M) - cov_diff_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) - - return cov_diff_diff_inv - - -def _multinormal_logpdf(Sigma_inv: torch.Tensor, x: torch.Tensor): - return ( - -0.5 * x @ Sigma_inv @ x - + 0.5 * torch.logdet(Sigma_inv) - - 0.5 * x.shape[0] * math.log(2 * math.pi) - ) + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / obs_noise_var + return cov_diff_diff_inv, logdet class _SampledGP(botorch.models.model.Model): def __init__( self, - kernel: gpytorch.kernels.Kernel, + kernel_func: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], x: torch.Tensor, preferences: torch.Tensor, obs_noise_var: torch.Tensor, diff: torch.Tensor, ) -> None: - self.kernel = kernel + super().__init__() + self.kernel_func = kernel_func self.x = x self.preferences = preferences self.diff = diff self.obs_noise_var = obs_noise_var - self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( + self._cov_diff_diff_inv, _ = _compute_cov_diff_diff_inv_and_logdet( preferences=preferences, - cov_x_x=self.kernel(x).to_dense(), + cov_x_x=self.kernel_func(x, x), obs_noise_var=float(obs_noise_var), ) def posterior( self, - x2: torch.Tensor, + X: torch.Tensor, output_indices: list[int] | None = None, observation_noise: bool = False, posterior_transform: Any | None = None, @@ -122,19 +125,15 @@ class _SampledGP(botorch.models.model.Model): ) -> botorch.posteriors.gpytorch.GPyTorchPosterior: assert posterior_transform is None assert output_indices is None - assert self.x.shape[-1] == x2.shape[-1] + assert self.x.shape[-1] == X.shape[-1] - x_expanded = self.x.expand(x2.shape[:-2] + (self.x.shape[-2], x2.shape[-1])) + x_expanded = self.x.expand(X.shape[:-2] + (self.x.shape[-2], X.shape[-1])) - cov_x2_x: torch.Tensor = self.kernel(x2, x_expanded).to_dense() - cov_x2_diff: torch.Tensor = ( - cov_x2_x[..., self.preferences[:, 0]] - cov_x2_x[..., self.preferences[:, 1]] - ) + cov_X_x = self.kernel_func(X, x_expanded) + cov_X_diff = cov_X_x[..., self.preferences[:, 0]] - cov_X_x[..., self.preferences[:, 1]] - mean: torch.Tensor = cov_x2_diff @ (self._cov_diff_diff_inv @ self.diff) - cov: torch.Tensor = self.kernel( - x2 - ).to_dense() - cov_x2_diff @ self._cov_diff_diff_inv @ cov_x2_diff.transpose(-1, -2) + mean = cov_X_diff @ (self._cov_diff_diff_inv @ self.diff) + cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose(-1, -2) if observation_noise: idx = torch.arange(cov.shape[-1]) cov[..., idx, idx] += self.obs_noise_var @@ -156,13 +155,21 @@ class _SampledGP(botorch.models.model.Model): class _PreferentialGP: - def _kernel_factory(self, lengthscale: torch.Tensor) -> gpytorch.kernels.Kernel: - kernel = gpytorch.kernels.MaternKernel( - nu=2.5, - ard_num_dims=lengthscale.shape[0], - ) - kernel.lengthscale = lengthscale - return kernel + def _kernel_func(self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, nu: float) -> torch.Tensor: + x1_ = x1.div(lengthscale) + x2_ = x2.div(lengthscale) + distance = torch.cdist(x1_, x2_) + exp_component = torch.exp(-math.sqrt(nu * 2) * distance) + + if nu == 0.5: + constant_component = 1 + elif nu == 1.5: + constant_component = (math.sqrt(3) * distance).add(1) + elif nu == 2.5: + constant_component = (math.sqrt(5) * distance).add(1).add(5.0 / 3.0 * distance**2) + else: + raise NotImplementedError(f"nu should be 0.5, 1.5 or 2.5") + return constant_component * exp_component def _potential_func( self, @@ -173,30 +180,38 @@ class _PreferentialGP: log_noise: torch.Tensor, ) -> torch.Tensor: lengthscale = torch.exp(log_lengthscale) - noise = torch.exp(log_noise) + noise = torch.exp(log_noise) + self.minimum_noise + log_transform_jacobian = torch.sum(log_lengthscale) + log_noise - log_prior = self.lengthscale_prior.log_prob(lengthscale) + self.noise_prior.log_prob(noise) - cov_inv = _compute_cov_diff_diff_inv( + log_prior = torch.sum(self.lengthscale_prior.log_prob(lengthscale)) + self.noise_prior.log_prob(noise) + cov_x_x = self._kernel_func(x, x, lengthscale, 2.5) + cov_inv, cov_inv_logdet = _compute_cov_diff_diff_inv_and_logdet( preferences=preferences, - cov_x_x=self._kernel_factory(lengthscale)(x).to_dense(), + cov_x_x=cov_x_x, obs_noise_var=noise, ) - log_likelihood = _multinormal_logpdf(cov_inv, diff) - return log_prior + log_likelihood + log_transform_jacobian + + log_likelihood = -0.5 * diff @ cov_inv @ diff + 0.5 * cov_inv_logdet + + return -(log_prior + log_transform_jacobian + log_likelihood) def __init__( self, lengthscale_prior: Prior, noise_prior: Prior, + minimum_lengthscale: float, + minimum_noise: float, dims: int, ) -> None: self.lengthscale_prior: Prior = lengthscale_prior.expand((dims,)) self.noise_prior = noise_prior + self.minimum_lengthscale = minimum_lengthscale + self.minimum_noise = minimum_noise self.dims = dims - self._x = torch.empty((0, dims), dtype=torch.float64) - self._preferences = torch.empty((0, 2), dtype=torch.float64) - self._diff = torch.empty((0,), dtype=torch.float64) + self._x = torch.empty((0, dims), dtype=torch.float64, requires_grad=False) + self._preferences = torch.empty((0, 2), dtype=torch.int32, requires_grad=False) + self._diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) initial_raw_params = { "log_lengthscale": torch.log(self.lengthscale_prior.sample()), @@ -204,14 +219,8 @@ class _PreferentialGP: } self._potential_func_jit = torch.jit.trace( - self._potential_func, - ( - self._x, - self._preferences, - self._diff, - initial_raw_params["log_lengthscale"], - initial_raw_params["log_noise"], - ), + self._potential_func, (self._x, self._preferences, self._diff, initial_raw_params["log_lengthscale"], initial_raw_params["log_noise"]), + check_trace=False, ) # HMC-Gibbs workarounds @@ -224,46 +233,61 @@ class _PreferentialGP: diff=self._diff, log_lengthscale=z["log_lengthscale"], log_noise=z["log_noise"], - ) + ), + adapt_step_size=True, + adapt_mass_matrix=False, + target_accept_prob=0.5, + step_size=0.1, ) self._nuts.initial_params = initial_raw_params - self._nuts.setup(warmup_steps=1e15) # Infinite warmup + self._nuts.setup(warmup_steps=1e15) # Use default step size self._last_params = initial_raw_params + def sample_gp(self, x: torch.Tensor, preferences: torch.Tensor, cycles: int) -> _SampledGP: if len(preferences) == 0: + lengthscale = self.lengthscale_prior.sample() + self.minimum_lengthscale + noise = self.noise_prior.sample() + self.minimum_noise return _SampledGP( - kernel=self._kernel_factory(self.lengthscale_prior.sample()), + kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale, 2.5), x=x, preferences=preferences, - obs_noise_var=self.noise_prior.sample(), + obs_noise_var=noise, diff=torch.empty((0,), dtype=torch.float64), ) else: + self._x = x + self._preferences = preferences + original_diff_size = len(self._diff) self._diff.resize_(len(preferences)) self._diff[original_diff_size:] = 0.0 + self._x.requires_grad_(False) + self._preferences.requires_grad_(False) + self._diff.requires_grad_(False) + for _ in range(cycles): - kernel = self._kernel_factory(torch.exp(self._last_params["log_lengthscale"])) - cov_diff_diff_inv = _compute_cov_diff_diff_inv( - preferences=preferences, - cov_x_x=kernel(x).to_dense(), - obs_noise_var=float(torch.exp(self._last_params["log_noise"])), - ) - self._diff = _orthants_MVN_Gibbs_sampling( - cov_inv=cov_diff_diff_inv, cycles=10, initial_sample=self._diff - )[:-1] + with torch.no_grad(): + cov_diff_diff_inv, _ = _compute_cov_diff_diff_inv_and_logdet( + preferences=preferences, + cov_x_x=self._kernel_func(x, x, torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale, nu=2.5), + obs_noise_var=torch.exp(self._last_params["log_noise"]) + self.minimum_noise, + ) + self._diff = _orthants_MVN_Gibbs_sampling_jit( + cov_inv=cov_diff_diff_inv, initial_sample=self._diff, cycles=10, + )[-1] self._nuts.clear_cache() - self._last_raw_params = self._nuts.sample(self._last_raw_params) - + self._last_params = self._nuts.sample(self._last_params) + lengthscale = torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale + noise = torch.exp(self._last_params["log_noise"]) + self.minimum_noise return _SampledGP( - kernel=self._kernel_factory(torch.exp(self._last_params["log_lengthscale"])), + kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale, 2.5), x=x, preferences=preferences, - obs_noise_var=torch.exp(self._last_params["log_noise"]), + obs_noise_var=noise, diff=self._diff, ) @@ -277,13 +301,14 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, ) -> None: - self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(3.0, 6.0) - self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(1.1, 10.0) + self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(5.0, 10.0) + self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) + + self._rng = np.random.RandomState(seed) self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( seed=self._rng.randint(2**32) ) - self._rng = np.random.RandomState(seed) self._search_space = optuna.search_space.IntersectionSearchSpace() self._gp: _PreferentialGP | None = None @@ -314,18 +339,19 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): search_space, transform_log=True, transform_step=True, transform_0_1=True ) params = torch.tensor( - [trans.transform(trials[t].params) for t in trials_with_preference], + np.array([trans.transform(trials[t].params) for t in trials_with_preference]), dtype=torch.float64, ) pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) - with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) pyro.set_rng_seed(self._rng.randint(2**32)) self._gp = self._gp or _PreferentialGP( lengthscale_prior=self.lengthscale_prior, + minimum_lengthscale=0.1, noise_prior=self.noise_prior, + minimum_noise=1e-6, # To avoid NaN dims=len(trans.bounds), ) if self._gp.dims != len(trans.bounds): From 81ea33368e986f2e8a2c2556250d930a2faaeef1 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Wed, 30 Aug 2023 10:05:43 +0900 Subject: [PATCH 004/104] format --- optuna_dashboard/preferential/samplers/_gp.py | 66 +++++++++++++------ 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/_gp.py b/optuna_dashboard/preferential/samplers/_gp.py index 2f8bdba6..7f1f4dda 100644 --- a/optuna_dashboard/preferential/samplers/_gp.py +++ b/optuna_dashboard/preferential/samplers/_gp.py @@ -1,15 +1,16 @@ -#%% +# %% from __future__ import annotations import math -from typing import Any, Callable +from typing import Any +from typing import Callable import botorch.acquisition.analytic import botorch.models.model import botorch.optim import botorch.posteriors.gpytorch -import gpytorch.kernels import gpytorch.constraints +import gpytorch.kernels from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna @@ -19,6 +20,7 @@ import torch from .._system_attrs import get_preferences + def _orthants_MVN_Gibbs_sampling( cov_inv: torch.Tensor, cycles: int, @@ -38,9 +40,7 @@ def _orthants_MVN_Gibbs_sampling( for j in range(dim): conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=-conditional_mean / conditional_std[j] - ) + _one_side_trunc_norm_sampling(lower=-conditional_mean / conditional_std[j]) * conditional_std[j] + conditional_mean ) @@ -48,6 +48,7 @@ def _orthants_MVN_Gibbs_sampling( return out + def _one_side_trunc_norm_sampling(lower: torch.Tensor) -> torch.Tensor: if lower > 4.0: r = torch.max(torch.tensor(1e-300), torch.rand(torch.Size(()), dtype=torch.float64)) @@ -61,7 +62,8 @@ def _one_side_trunc_norm_sampling(lower: torch.Tensor) -> torch.Tensor: _orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) - + + def _compute_cov_diff_diff_inv_and_logdet( preferences: torch.Tensor, cov_x_x: torch.Tensor, @@ -88,12 +90,13 @@ def _compute_cov_diff_diff_inv_and_logdet( cov_diff_diff_inv = ( cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] ) - cov_diff_diff_inv *= -1 / obs_noise_var ** 2 + cov_diff_diff_inv *= -1 / obs_noise_var**2 idx_M = torch.arange(M) cov_diff_diff_inv[idx_M, idx_M] += 1.0 / obs_noise_var return cov_diff_diff_inv, logdet + class _SampledGP(botorch.models.model.Model): def __init__( self, @@ -133,7 +136,9 @@ class _SampledGP(botorch.models.model.Model): cov_X_diff = cov_X_x[..., self.preferences[:, 0]] - cov_X_x[..., self.preferences[:, 1]] mean = cov_X_diff @ (self._cov_diff_diff_inv @ self.diff) - cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose(-1, -2) + cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose( + -1, -2 + ) if observation_noise: idx = torch.arange(cov.shape[-1]) cov[..., idx, idx] += self.obs_noise_var @@ -155,7 +160,9 @@ class _SampledGP(botorch.models.model.Model): class _PreferentialGP: - def _kernel_func(self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, nu: float) -> torch.Tensor: + def _kernel_func( + self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, nu: float + ) -> torch.Tensor: x1_ = x1.div(lengthscale) x2_ = x2.div(lengthscale) distance = torch.cdist(x1_, x2_) @@ -183,7 +190,9 @@ class _PreferentialGP: noise = torch.exp(log_noise) + self.minimum_noise log_transform_jacobian = torch.sum(log_lengthscale) + log_noise - log_prior = torch.sum(self.lengthscale_prior.log_prob(lengthscale)) + self.noise_prior.log_prob(noise) + log_prior = torch.sum( + self.lengthscale_prior.log_prob(lengthscale) + ) + self.noise_prior.log_prob(noise) cov_x_x = self._kernel_func(x, x, lengthscale, 2.5) cov_inv, cov_inv_logdet = _compute_cov_diff_diff_inv_and_logdet( preferences=preferences, @@ -219,7 +228,14 @@ class _PreferentialGP: } self._potential_func_jit = torch.jit.trace( - self._potential_func, (self._x, self._preferences, self._diff, initial_raw_params["log_lengthscale"], initial_raw_params["log_noise"]), + self._potential_func, + ( + self._x, + self._preferences, + self._diff, + initial_raw_params["log_lengthscale"], + initial_raw_params["log_noise"], + ), check_trace=False, ) @@ -233,7 +249,7 @@ class _PreferentialGP: diff=self._diff, log_lengthscale=z["log_lengthscale"], log_noise=z["log_noise"], - ), + ), adapt_step_size=True, adapt_mass_matrix=False, target_accept_prob=0.5, @@ -244,7 +260,6 @@ class _PreferentialGP: self._nuts.setup(warmup_steps=1e15) # Use default step size self._last_params = initial_raw_params - def sample_gp(self, x: torch.Tensor, preferences: torch.Tensor, cycles: int) -> _SampledGP: if len(preferences) == 0: lengthscale = self.lengthscale_prior.sample() + self.minimum_lengthscale @@ -272,16 +287,27 @@ class _PreferentialGP: with torch.no_grad(): cov_diff_diff_inv, _ = _compute_cov_diff_diff_inv_and_logdet( preferences=preferences, - cov_x_x=self._kernel_func(x, x, torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale, nu=2.5), - obs_noise_var=torch.exp(self._last_params["log_noise"]) + self.minimum_noise, + cov_x_x=self._kernel_func( + x, + x, + torch.exp(self._last_params["log_lengthscale"]) + + self.minimum_lengthscale, + nu=2.5, + ), + obs_noise_var=torch.exp(self._last_params["log_noise"]) + + self.minimum_noise, ) self._diff = _orthants_MVN_Gibbs_sampling_jit( - cov_inv=cov_diff_diff_inv, initial_sample=self._diff, cycles=10, + cov_inv=cov_diff_diff_inv, + initial_sample=self._diff, + cycles=10, )[-1] self._nuts.clear_cache() self._last_params = self._nuts.sample(self._last_params) - lengthscale = torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale + lengthscale = ( + torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale + ) noise = torch.exp(self._last_params["log_noise"]) + self.minimum_noise return _SampledGP( kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale, 2.5), @@ -303,7 +329,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): ) -> None: self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(5.0, 10.0) self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) - + self._rng = np.random.RandomState(seed) self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( seed=self._rng.randint(2**32) @@ -351,7 +377,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): lengthscale_prior=self.lengthscale_prior, minimum_lengthscale=0.1, noise_prior=self.noise_prior, - minimum_noise=1e-6, # To avoid NaN + minimum_noise=1e-6, # To avoid NaN dims=len(trans.bounds), ) if self._gp.dims != len(trans.bounds): From 857122d5b1063cffe1e5a6bdd00fefb4991ee1ef Mon Sep 17 00:00:00 2001 From: Contramundum Date: Wed, 30 Aug 2023 11:38:16 +0900 Subject: [PATCH 005/104] Fix matern 3/2 kernel --- optuna_dashboard/preferential/samplers/gp.py | 26 ++++++-------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 7f1f4dda..aec4c22a 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -161,22 +161,11 @@ class _SampledGP(botorch.models.model.Model): class _PreferentialGP: def _kernel_func( - self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, nu: float + self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, ) -> torch.Tensor: - x1_ = x1.div(lengthscale) - x2_ = x2.div(lengthscale) - distance = torch.cdist(x1_, x2_) - exp_component = torch.exp(-math.sqrt(nu * 2) * distance) - - if nu == 0.5: - constant_component = 1 - elif nu == 1.5: - constant_component = (math.sqrt(3) * distance).add(1) - elif nu == 2.5: - constant_component = (math.sqrt(5) * distance).add(1).add(5.0 / 3.0 * distance**2) - else: - raise NotImplementedError(f"nu should be 0.5, 1.5 or 2.5") - return constant_component * exp_component + # Matern 3/2 kernel + d = math.sqrt(3) * torch.cdist(x1 / lengthscale, x2 / lengthscale) + return torch.exp(-d) * (d + 1) def _potential_func( self, @@ -193,7 +182,7 @@ class _PreferentialGP: log_prior = torch.sum( self.lengthscale_prior.log_prob(lengthscale) ) + self.noise_prior.log_prob(noise) - cov_x_x = self._kernel_func(x, x, lengthscale, 2.5) + cov_x_x = self._kernel_func(x, x, lengthscale) cov_inv, cov_inv_logdet = _compute_cov_diff_diff_inv_and_logdet( preferences=preferences, cov_x_x=cov_x_x, @@ -265,7 +254,7 @@ class _PreferentialGP: lengthscale = self.lengthscale_prior.sample() + self.minimum_lengthscale noise = self.noise_prior.sample() + self.minimum_noise return _SampledGP( - kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale, 2.5), + kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale), x=x, preferences=preferences, obs_noise_var=noise, @@ -292,7 +281,6 @@ class _PreferentialGP: x, torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale, - nu=2.5, ), obs_noise_var=torch.exp(self._last_params["log_noise"]) + self.minimum_noise, @@ -310,7 +298,7 @@ class _PreferentialGP: ) noise = torch.exp(self._last_params["log_noise"]) + self.minimum_noise return _SampledGP( - kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale, 2.5), + kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale), x=x, preferences=preferences, obs_noise_var=noise, From 012b71a42a90f2e136e3605f2c7cf186f137a646 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Wed, 30 Aug 2023 12:56:50 +0900 Subject: [PATCH 006/104] format --- optuna_dashboard/preferential/samplers/gp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index aec4c22a..09b64120 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -161,7 +161,10 @@ class _SampledGP(botorch.models.model.Model): class _PreferentialGP: def _kernel_func( - self, x1: torch.Tensor, x2: torch.Tensor, lengthscale: torch.Tensor, + self, + x1: torch.Tensor, + x2: torch.Tensor, + lengthscale: torch.Tensor, ) -> torch.Tensor: # Matern 3/2 kernel d = math.sqrt(3) * torch.cdist(x1 / lengthscale, x2 / lengthscale) From ee02294381f649e4a2853ce1c25e2f9d81c55d57 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 31 Aug 2023 12:17:14 +0900 Subject: [PATCH 007/104] Implement EP for optimizing hyperparameters --- optuna_dashboard/preferential/samplers/gp.py | 334 +++++++++---------- 1 file changed, 154 insertions(+), 180 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 09b64120..63abedf3 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,4 +1,3 @@ -# %% from __future__ import annotations import math @@ -15,17 +14,13 @@ from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna import optuna._transform -import pyro.infer.mcmc import torch +from torch import Tensor from .._system_attrs import get_preferences -def _orthants_MVN_Gibbs_sampling( - cov_inv: torch.Tensor, - cycles: int, - initial_sample: torch.Tensor, -) -> torch.Tensor: +def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: Tensor) -> Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) @@ -49,9 +44,9 @@ def _orthants_MVN_Gibbs_sampling( return out -def _one_side_trunc_norm_sampling(lower: torch.Tensor) -> torch.Tensor: +def _one_side_trunc_norm_sampling(lower: Tensor) -> Tensor: if lower > 4.0: - r = torch.max(torch.tensor(1e-300), torch.rand(torch.Size(()), dtype=torch.float64)) + r = torch.clamp_min(torch.rand(torch.Size(()), dtype=torch.float64), min=1e-300) return (lower * lower - 2 * r.log()).sqrt() else: SQRT2 = math.sqrt(2) @@ -64,63 +59,53 @@ def _one_side_trunc_norm_sampling(lower: torch.Tensor) -> torch.Tensor: _orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) -def _compute_cov_diff_diff_inv_and_logdet( - preferences: torch.Tensor, - cov_x_x: torch.Tensor, - obs_noise_var: torch.Tensor, -) -> tuple[torch.Tensor, torch.Tensor]: +def _compute_cov_diff_diff_inv(preferences: Tensor, cov_x_x: Tensor, noise_var: Tensor) -> Tensor: N = cov_x_x.shape[0] M = preferences.shape[0] # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) - # det(sI + A K A^T) = s^N det(I + s^-1 A K A^T) = s^N det(I + s^-1 A^T A K) - I_plus_sinv_AT_A_K = torch.eye(N, dtype=torch.float64) A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] - I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / obs_noise_var)) - I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / obs_noise_var)) - lu, piv = torch.linalg.lu_factor(I_plus_sinv_AT_A_K) - - logdet = -(lu.diagonal().abs() * obs_noise_var).log().sum() - - schur_inv: torch.Tensor = torch.linalg.lu_solve(lu, piv, cov_x_x, left=False) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / noise_var)) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / noise_var)) + schur_inv: Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] cov_diff_diff_inv = ( cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] ) - cov_diff_diff_inv *= -1 / obs_noise_var**2 + cov_diff_diff_inv *= -1 / noise_var**2 idx_M = torch.arange(M) - cov_diff_diff_inv[idx_M, idx_M] += 1.0 / obs_noise_var + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / noise_var - return cov_diff_diff_inv, logdet + return cov_diff_diff_inv class _SampledGP(botorch.models.model.Model): def __init__( self, - kernel_func: Callable[[torch.Tensor, torch.Tensor], torch.Tensor], - x: torch.Tensor, - preferences: torch.Tensor, - obs_noise_var: torch.Tensor, - diff: torch.Tensor, + kernel_func: Callable[[Tensor, Tensor], Tensor], + x: Tensor, + preferences: Tensor, + noise_var: Tensor, + diff: Tensor, ) -> None: super().__init__() self.kernel_func = kernel_func self.x = x self.preferences = preferences self.diff = diff - self.obs_noise_var = obs_noise_var - self._cov_diff_diff_inv, _ = _compute_cov_diff_diff_inv_and_logdet( + self.noise_var = noise_var + self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( preferences=preferences, cov_x_x=self.kernel_func(x, x), - obs_noise_var=float(obs_noise_var), + noise_var=noise_var, ) def posterior( self, - X: torch.Tensor, + X: Tensor, output_indices: list[int] | None = None, observation_noise: bool = False, posterior_transform: Any | None = None, @@ -141,7 +126,7 @@ class _SampledGP(botorch.models.model.Model): ) if observation_noise: idx = torch.arange(cov.shape[-1]) - cov[..., idx, idx] += self.obs_noise_var + cov[..., idx, idx] += self.noise_var return botorch.posteriors.gpytorch.GPyTorchPosterior( distribution=gpytorch.distributions.MultivariateNormal( @@ -159,153 +144,135 @@ class _SampledGP(botorch.models.model.Model): return 1 +def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]: + SQRT_HALF = math.sqrt(0.5) + SQRT_HALF_PI = math.sqrt(0.5 * math.pi) + logz = torch.special.log_ndtr(-alpha) + mean = 1 / (SQRT_HALF_PI * torch.special.erfcx(alpha * SQRT_HALF)) + var = 1 - mean * (mean - alpha) + return (mean, var, logz) + + +def _orthants_MVN_EP( + cov0: Tensor, preferences: Tensor, noise_var: Tensor, cycles: int +) -> tuple[Tensor, Tensor, Tensor]: + N = cov0.shape[0] + M = preferences.shape[0] + mu = torch.zeros(N, dtype=cov0.dtype) + cov = cov0.clone() + virtual_obs_a = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + virtual_obs_b = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + log_zs = torch.zeros(M, dtype=cov0.dtype) + + for _ in range(cycles): + for i in range(M): + pref_i = preferences[i, :] + mean1 = mu[pref_i[0]] - mu[pref_i[1]] + Sxy = cov[pref_i[0]] - cov[pref_i[1]] + var1 = Sxy[pref_i[0]] - Sxy[pref_i[1]] + + r0 = (1 - var1 * virtual_obs_a[i]).reciprocal() + var0 = var1 * r0 + mean0 = (mean1 + var1 * virtual_obs_b[i]) * r0 + + obs_var = var0 + noise_var + obs_sigma = torch.sqrt(obs_var) + alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20) + mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha) + + kalman_factor = var0 / torch.clamp_min(obs_var, min=1e-20) + mean2 = mean0 + obs_sigma * mean_norm * kalman_factor + var2 = kalman_factor * (noise_var + var_norm * var0) + + var1_var2_inv = torch.clamp_min(var1 * var2, min=1e-20).reciprocal() + db = (mean1 * var2 - mean2 * var1) * var1_var2_inv + da = (var1 - var2) * var1_var2_inv + virtual_obs_b[i] = virtual_obs_b[i] + db + virtual_obs_a[i] = virtual_obs_a[i] + da + + dr = (1 + var1 * da).reciprocal() + mu = mu - Sxy * ((db + mean1 * da) * dr) + cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :] + log_zs[i] = logz + return (mu, cov, torch.sum(log_zs)) + + +_orthants_MVN_EP_jit = torch.jit.script(_orthants_MVN_EP) + + class _PreferentialGP: - def _kernel_func( - self, - x1: torch.Tensor, - x2: torch.Tensor, - lengthscale: torch.Tensor, - ) -> torch.Tensor: - # Matern 3/2 kernel - d = math.sqrt(3) * torch.cdist(x1 / lengthscale, x2 / lengthscale) - return torch.exp(-d) * (d + 1) - - def _potential_func( - self, - x: torch.Tensor, - preferences: torch.Tensor, - diff: torch.Tensor, - log_lengthscale: torch.Tensor, - log_noise: torch.Tensor, - ) -> torch.Tensor: - lengthscale = torch.exp(log_lengthscale) - noise = torch.exp(log_noise) + self.minimum_noise - - log_transform_jacobian = torch.sum(log_lengthscale) + log_noise - log_prior = torch.sum( - self.lengthscale_prior.log_prob(lengthscale) - ) + self.noise_prior.log_prob(noise) - cov_x_x = self._kernel_func(x, x, lengthscale) - cov_inv, cov_inv_logdet = _compute_cov_diff_diff_inv_and_logdet( - preferences=preferences, - cov_x_x=cov_x_x, - obs_noise_var=noise, - ) - - log_likelihood = -0.5 * diff @ cov_inv @ diff + 0.5 * cov_inv_logdet - - return -(log_prior + log_transform_jacobian + log_likelihood) - - def __init__( - self, - lengthscale_prior: Prior, - noise_prior: Prior, - minimum_lengthscale: float, - minimum_noise: float, - dims: int, - ) -> None: - self.lengthscale_prior: Prior = lengthscale_prior.expand((dims,)) + def __init__(self, kernel: gpytorch.kernels.Kernel, noise_prior: Prior, dims: int) -> None: + self.kernel = kernel self.noise_prior = noise_prior - self.minimum_lengthscale = minimum_lengthscale - self.minimum_noise = minimum_noise self.dims = dims - self._x = torch.empty((0, dims), dtype=torch.float64, requires_grad=False) - self._preferences = torch.empty((0, 2), dtype=torch.int32, requires_grad=False) - self._diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) - - initial_raw_params = { - "log_lengthscale": torch.log(self.lengthscale_prior.sample()), - "log_noise": torch.log(self.noise_prior.sample()), - } - - self._potential_func_jit = torch.jit.trace( - self._potential_func, - ( - self._x, - self._preferences, - self._diff, - initial_raw_params["log_lengthscale"], - initial_raw_params["log_noise"], - ), - check_trace=False, + self.diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) + self.log_noise = torch.nn.Parameter( + torch.tensor(0.0, dtype=torch.float64), requires_grad=True ) - # HMC-Gibbs workarounds - # https://github.com/pyro-ppl/pyro/issues/1926 - - self._nuts = pyro.infer.mcmc.NUTS( - potential_fn=lambda z: self._potential_func_jit( - x=self._x, - preferences=self._preferences, - diff=self._diff, - log_lengthscale=z["log_lengthscale"], - log_noise=z["log_noise"], - ), - adapt_step_size=True, - adapt_mass_matrix=False, - target_accept_prob=0.5, - step_size=0.1, - ) - - self._nuts.initial_params = initial_raw_params - self._nuts.setup(warmup_steps=1e15) # Use default step size - self._last_params = initial_raw_params - - def sample_gp(self, x: torch.Tensor, preferences: torch.Tensor, cycles: int) -> _SampledGP: + def fit_params_EP(self, X: Tensor, preferences: Tensor) -> None: if len(preferences) == 0: - lengthscale = self.lengthscale_prior.sample() + self.minimum_lengthscale - noise = self.noise_prior.sample() + self.minimum_noise + return + tolerance = 1e-3 + max_iter = 100 + + optim = torch.optim.LBFGS([*self.kernel.parameters(), self.log_noise]) + + last_params = [p.detach().clone() for p in optim.param_groups[0]["params"]] + for _ in range(max_iter): + + def closure(): + optim.zero_grad() + noise = self.log_noise.exp() + cov0 = self.kernel.forward(X, X).to_dense() + _, _, logz = _orthants_MVN_EP_jit(cov0, preferences, noise, cycles=2) + + loss = -logz - self.noise_prior.log_prob(noise) + for _, _, prior, param, _ in self.kernel.named_priors(): + loss = loss - prior.log_prob(param(self.kernel)).sum() + + loss.backward() + return loss + + optim.step(closure) + + # Check for convergence + params = optim.param_groups[0]["params"] + for p_old, p_new in zip(last_params, params): + if torch.max(torch.abs(p_old - p_new)) > tolerance: + break + else: + break + last_params = [p.detach().clone() for p in params] + + def sample_gp(self, x: Tensor, preferences: Tensor) -> _SampledGP: + self.fit_params_EP(x, preferences) + print({name: p.exp() for name, p in self.kernel.named_parameters()}) + print({"noise": self.log_noise.exp()}) + + with torch.no_grad(): + cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel(x, x).to_dense(), + noise_var=self.log_noise.exp(), + ) + + original_diff_size = len(self.diff) + self.diff.resize_(len(preferences)) + self.diff[original_diff_size:] = 0.0 + + self.diff = _orthants_MVN_Gibbs_sampling_jit( + cov_inv=cov_diff_diff_inv, + initial_sample=self.diff, + cycles=20, + )[-1] return _SampledGP( - kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale), + kernel_func=lambda x1, x2: self.kernel(x1, x2).to_dense(), x=x, preferences=preferences, - obs_noise_var=noise, - diff=torch.empty((0,), dtype=torch.float64), - ) - else: - self._x = x - self._preferences = preferences - - original_diff_size = len(self._diff) - self._diff.resize_(len(preferences)) - self._diff[original_diff_size:] = 0.0 - - self._x.requires_grad_(False) - self._preferences.requires_grad_(False) - self._diff.requires_grad_(False) - - for _ in range(cycles): - with torch.no_grad(): - cov_diff_diff_inv, _ = _compute_cov_diff_diff_inv_and_logdet( - preferences=preferences, - cov_x_x=self._kernel_func( - x, - x, - torch.exp(self._last_params["log_lengthscale"]) - + self.minimum_lengthscale, - ), - obs_noise_var=torch.exp(self._last_params["log_noise"]) - + self.minimum_noise, - ) - - self._diff = _orthants_MVN_Gibbs_sampling_jit( - cov_inv=cov_diff_diff_inv, - initial_sample=self._diff, - cycles=10, - )[-1] - self._nuts.clear_cache() - self._last_params = self._nuts.sample(self._last_params) - lengthscale = ( - torch.exp(self._last_params["log_lengthscale"]) + self.minimum_lengthscale - ) - noise = torch.exp(self._last_params["log_noise"]) + self.minimum_noise - return _SampledGP( - kernel_func=lambda x1, x2: self._kernel_func(x1, x2, lengthscale), - x=x, - preferences=preferences, - obs_noise_var=noise, - diff=self._diff, + noise_var=self.log_noise.exp(), + diff=self.diff, ) @@ -313,12 +280,12 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): def __init__( self, *, - lengthscale_prior: Prior | None = None, + kernel: gpytorch.kernels.Kernel | None = None, noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, ) -> None: - self.lengthscale_prior = lengthscale_prior or gpytorch.priors.GammaPrior(5.0, 10.0) + self.kernel = kernel self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) self._rng = np.random.RandomState(seed) @@ -362,13 +329,20 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) - pyro.set_rng_seed(self._rng.randint(2**32)) self._gp = self._gp or _PreferentialGP( - lengthscale_prior=self.lengthscale_prior, - minimum_lengthscale=0.1, + kernel=self.kernel + or gpytorch.kernels.MaternKernel( + nu=1.5, + ard_num_dims=len(trans.bounds), + lengthscale_prior=gpytorch.priors.GammaPrior(5.0, 10.0), + lengthscale_constraint=gpytorch.constraints.GreaterThan( + 0.0, + transform=torch.exp, + inv_transform=torch.log, + ), + ), noise_prior=self.noise_prior, - minimum_noise=1e-6, # To avoid NaN dims=len(trans.bounds), ) if self._gp.dims != len(trans.bounds): @@ -377,7 +351,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): "Dynamic search space is not supported in PreferentialGPSampler." ) - sampled_gp = self._gp.sample_gp(params, pref_ids, cycles=10) + sampled_gp = self._gp.sample_gp(params, pref_ids) acqf = botorch.acquisition.analytic.LogExpectedImprovement( model=sampled_gp, best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), From 2e05d2c53d51df9afc7267dbb7cbccced30ef5ff Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 31 Aug 2023 12:24:01 +0900 Subject: [PATCH 008/104] Fix mypy error --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 63abedf3..bc59c200 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -222,7 +222,7 @@ class _PreferentialGP: last_params = [p.detach().clone() for p in optim.param_groups[0]["params"]] for _ in range(max_iter): - def closure(): + def closure() -> Tensor: optim.zero_grad() noise = self.log_noise.exp() cov0 = self.kernel.forward(X, X).to_dense() From 516f06a19c49fe674336b4f74575eec64e4b74dc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 31 Aug 2023 14:08:05 +0900 Subject: [PATCH 009/104] wip --- .../ts/components/PreferentialTrials.tsx | 138 +++- optuna_dashboard/ts/components/TrialList.tsx | 727 ++++++++++-------- optuna_dashboard/ts/state.ts | 11 + 3 files changed, 535 insertions(+), 341 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 5c141403..1129d2a0 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,17 +7,59 @@ import { CardContent, CardActions, CardActionArea, + CardMedia, + MenuItem, + Select, + FormControl, + FormLabel, + TextField, + Modal, } from "@mui/material" -import ClearIcon from "@mui/icons-material/Clear" -import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" -import Modal from "@mui/material/Modal" -import { red } from "@mui/material/colors" - +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" +import SettingsIcon from "@mui/icons-material/Settings" +import red from "@mui/material/colors/red" +import { useRecoilValue, useSetRecoilState } from "recoil" import { actionCreator } from "../action" -import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" +import { + feedbackComponent, + FeedbackComponentType, + feedbackArtifactKey, +} from "../state" +import { + TrialArtifactActions, + TrialArtifactContent, + TrialListDetail, +} from "./TrialList" + +const FeedbackContent: FC<{ + trial: Trial + artifact?: Artifact +}> = ({ trial, artifact }) => { + const componentId = useRecoilValue(feedbackComponent) + + if (componentId === "note") { + return + } + if (componentId === "artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + + return null +} const PreferentialTrial: FC<{ trial?: Trial @@ -29,6 +71,9 @@ const PreferentialTrial: FC<{ const trialWidth = 500 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) + const componentId = useRecoilValue(feedbackComponent) + const artifactKey = useRecoilValue(feedbackArtifactKey) + const artifact = trial?.artifacts.find((a) => a.filename === artifactKey) if (trial == undefined) { return ( @@ -52,7 +97,19 @@ const PreferentialTrial: FC<{ }} > - Trial {trial.number} + + Trial {trial.number} + {componentId === "artifact" && artifact !== undefined + ? ` (${artifact.filename})` + : ""} + + {componentId === "artifact" && artifact !== undefined ? ( + + ) : null} - + = ({ numbers: studyDetail.best_trials.map((t) => t.number), last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), }) + const [settingShown, setSettingShown] = useState(false) + const outputComponent = useRecoilValue(feedbackComponent) + const setOutputComponent = useSetRecoilState(feedbackComponent) + const outputartifactKey = useRecoilValue(feedbackArtifactKey) + const setOutputartifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -228,7 +290,23 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ } return ( - + + setSettingShown(true)} + > + + = ({ /> ))} + + + Settings + + + Output Component: + + + {outputComponent === "artifact" ? ( + + Output File: + { + setOutputartifactKey(e.target.value) + }} + /> + + ) : null} + + + ) } diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 4aceb619..25ec9fab 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -22,6 +22,7 @@ import { CardActionArea, Modal, } from "@mui/material" +import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -319,16 +320,397 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { +export const TrialArtifactContent: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + ) + } else { + return ( + + + + ) + } +} + +export const TrialArtifactActions: FC<{ + trial: Trial + artifact: Artifact + sx: SxProps +}> = ({ trial, artifact, sx }) => { + const [open3dModelViewer, setOpen3dModelViewer] = useState(false) + + if (artifact.mimetype.startsWith("image")) { + return null + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + <> + { + setOpen3dModelViewer(true) + }} + > + + + { + setOpen3dModelViewer(false) + }} + > + + + + + + ) + } + return null +} + +const TrialArtifact: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + const [openDeleteArtifactDialog, _] = useDeleteArtifactDialog() + const theme = useTheme() + if (artifact.mimetype.startsWith("image")) { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + + {artifact.filename} + + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } else { + return ( + + + + + {artifact.filename} + + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + + + + + + ) + } +} + +const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() + const [_, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) const [open3dModelViewer, setOpen3dModelViewer] = useState<{ [key: string]: boolean @@ -382,334 +764,15 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { Artifacts - {trial.artifacts.map((a) => { - if (a.mimetype.startsWith("image")) { - return ( - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = true - return obj - }) - }} - > - - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = false - return obj - }) - }} - > - - - - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } - })} + {trial.artifacts.map((a) => ( + + ))} {trial.state === "Running" || trial.state === "Waiting" ? ( ({ default: false, }) +export type FeedbackComponentType = "note" | "artifact" +export const feedbackComponent = atom({ + key: "feedbackComponent", + default: "note", +}) + +export const feedbackArtifactKey = atom({ + key: "feedbackArtifactKey", + default: "", +}) + export const useStudyDetailValue = (studyId: number): StudyDetail | null => { const studyDetails = useRecoilValue(studyDetailsState) return studyDetails[studyId] || null From 181319e89cfe0deef3b7deb02723d88a08772c21 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 31 Aug 2023 15:24:08 +0900 Subject: [PATCH 010/104] add setting to frontend --- .../ts/components/PreferentialTrials.tsx | 136 +++++++++++------- optuna_dashboard/ts/components/TrialList.tsx | 7 +- 2 files changed, 89 insertions(+), 54 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 1129d2a0..06bdad9d 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,7 +7,6 @@ import { CardContent, CardActions, CardActionArea, - CardMedia, MenuItem, Select, FormControl, @@ -61,6 +60,43 @@ const FeedbackContent: FC<{ return null } +const ModalPage: FC<{ + children: React.ReactNode + displayFlag: boolean + setDisplayFlag: (flag: boolean) => void +}> = ({ children, displayFlag, setDisplayFlag }) => { + const theme = useTheme() + return ( + setDisplayFlag(false)}> + + + {children} + + + + ) +} + const PreferentialTrial: FC<{ trial?: Trial studyDetail: StudyDetail @@ -73,7 +109,8 @@ const PreferentialTrial: FC<{ const [detailShown, setDetailShown] = useState(false) const componentId = useRecoilValue(feedbackComponent) const artifactKey = useRecoilValue(feedbackArtifactKey) - const artifact = trial?.artifacts.find((a) => a.filename === artifactKey) + const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) if (trial == undefined) { return ( @@ -97,12 +134,17 @@ const PreferentialTrial: FC<{ }} > - - Trial {trial.number} - {componentId === "artifact" && artifact !== undefined - ? ` (${artifact.filename})` - : ""} - + Trial {trial.number} + {componentId === "artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} {componentId === "artifact" && artifact !== undefined ? ( - setDetailShown(false)}> - - - true} - directions={[]} - objectiveNames={[]} - /> - - - + + true} + directions={[]} + objectiveNames={[]} + /> + ) } @@ -249,8 +267,8 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const [settingShown, setSettingShown] = useState(false) const outputComponent = useRecoilValue(feedbackComponent) const setOutputComponent = useSetRecoilState(feedbackComponent) - const outputartifactKey = useRecoilValue(feedbackArtifactKey) - const setOutputartifactKey = useSetRecoilState(feedbackArtifactKey) + const outputArtifactKey = useRecoilValue(feedbackArtifactKey) + const setOutputArtifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -357,14 +375,34 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {outputComponent === "artifact" ? ( - - Output File: - { - setOutputartifactKey(e.target.value) + + + User Attribute Key Corresponding to Output Artifact Id: + + ) : null} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 25ec9fab..f73a97e1 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -459,7 +459,7 @@ const TrialArtifact: FC<{ width: string height: string }> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog, _] = useDeleteArtifactDialog() + const [openDeleteArtifactDialog] = useDeleteArtifactDialog() const theme = useTheme() if (artifact.mimetype.startsWith("image")) { return ( @@ -710,11 +710,8 @@ const TrialArtifact: FC<{ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [_, renderDeleteArtifactDialog] = useDeleteArtifactDialog() + const [, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) - const [open3dModelViewer, setOpen3dModelViewer] = useState<{ - [key: string]: boolean - }>({}) const width = "200px" const height = "150px" From dbffb4cb6b48ff586f3d434ad3eab20bfe47708a Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 16:33:57 +0900 Subject: [PATCH 011/104] add undo --- optuna_dashboard/_app.py | 16 ++++ optuna_dashboard/_preferential_history.py | 76 ++++++++++++++----- optuna_dashboard/_serializer.py | 4 +- .../preferential/_system_attrs.py | 5 ++ optuna_dashboard/ts/action.ts | 16 ++++ optuna_dashboard/ts/apiClient.ts | 16 ++++ .../ts/components/PreferenceHistory.tsx | 52 ++++++++++--- optuna_dashboard/ts/types/index.d.ts | 1 + python_tests/test_preferential_history.py | 4 +- 9 files changed, 158 insertions(+), 32 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index f582d5e5..0b69dfc7 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preferential_history import report_history +from ._preferential_history import switching_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -296,6 +297,21 @@ def create_app( response.status = 204 return {} + @app.put("/api/studies//preference/") + @json_api_view + def switch_preference(study_id: int, history_uuid: str) -> dict[str, Any]: + try: + enable = request.json.get("enable", None) + if enable is None or not isinstance(enable, bool): + raise ValueError + except ValueError: + response.status = 400 + return {"reason": "Invalid request."} + switching_history(study_id, storage, history_uuid, enable) + + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 5c930932..dff10885 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -1,8 +1,8 @@ from __future__ import annotations from dataclasses import dataclass +from dataclasses import field from datetime import datetime -import json from typing import Any from typing import Literal from typing import TYPE_CHECKING @@ -10,6 +10,8 @@ import uuid from optuna.storages import BaseStorage +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +from .preferential._system_attrs import get_preference from .preferential._system_attrs import report_preferences @@ -28,7 +30,7 @@ if TYPE_CHECKING: ) -@dataclass(frozen=True) +@dataclass class ChooseWorstHistory: mode: Literal["ChooseWorst"] uuid: str @@ -36,6 +38,8 @@ class ChooseWorstHistory: timestamp: datetime candidates: list[int] # a list of trial number clicked: int # The worst trial number in the candidates. + evacuated_preference: list[tuple[int, int]] = field(default_factory=list) + # When undo the preference, this is used. Otherwise, this must be empty. def to_dict(self) -> dict[str, Any]: return { @@ -45,6 +49,8 @@ class ChooseWorstHistory: "timestamp": self.timestamp.isoformat(), "candidates": self.candidates, "clicked": self.clicked, + "enabled": len(self.evacuated_preference) == 0, + "evacuated_preference": self.evacuated_preference, } @@ -87,31 +93,65 @@ def report_history( storage.set_study_system_attr( study_id=study_id, key=key, - value=json.dumps(history.to_dict()), + value=history.to_dict(), ) -def serialize_preference_history( +def _load_preference_history(value: Any) -> History: + choice: dict[str, Any] = value + if choice["mode"] == "ChooseWorst": + return ChooseWorstHistory( + mode="ChooseWorst", + uuid=choice["uuid"], + preference_uuid=choice["preference_uuid"], + timestamp=datetime.fromisoformat(choice["timestamp"]), + candidates=choice["candidates"], + clicked=choice["clicked"], + evacuated_preference=choice["evacuated_preference"], + ) + else: + assert False, f"Unknown mode: {choice['mode']}" + + +def load_preference_history( + uuid: str, + system_attrs: dict[str, Any], +) -> History: + value = system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, []) + return _load_preference_history(value) + + +def serialize_preference_histories( system_attrs: dict[str, Any], ) -> list[dict[str, Any]]: histories: list[History] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue - choice: dict[str, Any] = json.loads(v) - if choice["mode"] == "ChooseWorst": - histories.append( - ChooseWorstHistory( - mode="ChooseWorst", - uuid=choice["uuid"], - preference_uuid=choice["preference_uuid"], - timestamp=datetime.fromisoformat(choice["timestamp"]), - candidates=choice["candidates"], - clicked=choice["clicked"], - ) - ) - else: - assert False, f"Unknown mode: {choice['mode']}" + histories.append(_load_preference_history(v)) histories.sort(key=lambda c: c.timestamp) return [history.to_dict() for history in histories] + + +def switching_history(study_id: int, storage: BaseStorage, uuid: str, enable: bool) -> None: + system_attrs = storage.get_study_system_attrs(study_id) + history = load_preference_history(uuid, system_attrs) + preference = get_preference(study_id, storage, history.preference_uuid) + print(history, preference, enable) + if enable and (len(preference) > 0 or len(history.evacuated_preference) == 0): + return + if (not enable) and (len(preference) == 0 or len(history.evacuated_preference) > 0): + return + history.evacuated_preference, preference = preference, history.evacuated_preference + print(history.to_dict(), preference) + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_PREFIX_HISTORY + history.uuid, + value=history.to_dict(), + ) + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_PREFIX_PREFERENCE + history.preference_uuid, + value=preference, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 19ceb19b..fc040424 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -14,7 +14,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preferential_history import serialize_preference_history +from ._preferential_history import serialize_preference_histories from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -157,7 +157,7 @@ def serialize_study_detail( if form_widgets: serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: - serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["preference_history"] = serialize_preference_histories(system_attrs) return serialized diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 00347d7e..4e3f55f5 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -34,6 +34,11 @@ def report_preferences( return preference_uuid +def get_preference(study_id: int, storage: BaseStorage, uuid: str) -> list[tuple[int, int]]: + system_attrs = storage.get_study_system_attrs(study_id) + return system_attrs.get(_SYSTEM_ATTR_PREFIX_PREFERENCE + uuid, []) # type: ignore + + def get_preferences( study_id: int, storage: BaseStorage, diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index e1896178..f42a17bb 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -16,6 +16,7 @@ import { deleteArtifactAPI, reportPreferenceAPI, skipPreferentialTrialAPI, + switchPreferentialHistoryAPI, } from "./apiClient" import { graphVisibilityState, @@ -609,6 +610,20 @@ export const actionCreator = () => { }) } + const switchPreferentialHistory = ( + studyId: number, + historyUuid: string, + enable: boolean + ) => { + switchPreferentialHistoryAPI(studyId, historyUuid, enable).catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to switch history. Reason: ${reason}`, { + variant: "error", + }) + console.log(err) + }) + } + return { updateAPIMeta, updateStudyDetail, @@ -630,6 +645,7 @@ export const actionCreator = () => { saveTrialUserAttrs, updatePreference, skipPreferentialTrial, + switchPreferentialHistory, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 9c72f739..6886f8b4 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -61,6 +61,7 @@ interface PreferenceChoiceResponce { clicked: number mode: PreferenceFeedbackMode timestamp: string + enabled: boolean } const convertPreferenceChoice = ( @@ -72,6 +73,7 @@ const convertPreferenceChoice = ( clicked: res.clicked, feedback_mode: res.mode, timestamp: new Date(res.timestamp), + enabled: res.enabled, } } @@ -362,3 +364,17 @@ export const skipPreferentialTrialAPI = ( return }) } + +export const switchPreferentialHistoryAPI = ( + studyId: number, + historyUuid: string, + enable: boolean +): Promise => { + return axiosInstance + .put(`/api/studies/${studyId}/preference/${historyUuid}`, { + enable: enable, + }) + .then(() => { + return + }) +} diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index e95a4cb1..76725faa 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -9,12 +9,15 @@ import { } from "@mui/material" import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" +import UndoIcon from "@mui/icons-material/Undo" +import RedoIcon from "@mui/icons-material/Redo" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import Modal from "@mui/material/Modal" import { TrialListDetail } from "./TrialList" import { MarkdownRenderer } from "./Note" import { red } from "@mui/material/colors" +import { actionCreator } from "../action" type TrialType = "worst" | "none" @@ -133,24 +136,52 @@ const CandidateTrial: FC<{ ) } -const ChoiceTrials: FC<{ choice: PreferenceChoice; trials: Trial[] }> = ({ - choice, - trials, -}) => { +const ChoiceTrials: FC<{ + choice: PreferenceChoice + trials: Trial[] + study_id: number +}> = ({ choice, trials, study_id }) => { const theme = useTheme() const worst_trials = new Set([choice.clicked]) + const actions = actionCreator() + const handleUndo = () => { + actions.switchPreferentialHistory(study_id, choice.uuid, false) + } + const handleRedo = () => { + actions.switchPreferentialHistory(study_id, choice.uuid, true) + } return ( - - {choice.timestamp.toISOString()} - + + {choice.timestamp.toLocaleString()} + + + + + + + + = ({ key={choice.uuid} choice={choice} trials={studyDetail.trials} + study_id={studyDetail.id} /> ))} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index da90ad1d..26c08d38 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -215,4 +215,5 @@ type PreferenceChoice = { clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date + enabled: boolean } diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 64591783..85f25374 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -3,7 +3,7 @@ from __future__ import annotations from typing import Callable from optuna_dashboard._preferential_history import report_history -from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard._serializer import serialize_preference_histories from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -40,7 +40,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) "clicked": 0, }, ) - history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + history = serialize_preference_histories(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2] From f4dea90e07f0403c36a86ee8ca75fe545696f75c Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 1 Sep 2023 17:27:14 +0900 Subject: [PATCH 012/104] add test --- optuna_dashboard/_preferential_history.py | 3 +- python_tests/test_api.py | 71 +++++++++++++++++++++++ python_tests/test_preferential_history.py | 58 ++++++++++++++++++ 3 files changed, 131 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index dff10885..f95d57d3 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -61,7 +61,7 @@ def report_history( study_id: int, storage: BaseStorage, input_data: NewHistoryJSON, -) -> None: +) -> str: preferences = [] if input_data["mode"] == "ChooseWorst": preferences = [ @@ -95,6 +95,7 @@ def report_history( key=key, value=history.to_dict(), ) + return history_uuid def _load_preference_history(value: Any) -> History: diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 7d732e37..1fc92508 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preferential_history import serialize_preference_histories from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -181,6 +182,76 @@ class APITestCase(TestCase): assert best_trials[0].number == 0 assert best_trials[1].number == 2 + def test_undo_redo_history(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 2, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert histories[0]["enabled"] + + history_uuid = histories[0]["uuid"] + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference/{history_uuid}", + "PUT", + body=json.dumps( + { + "enable": False, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert not histories[0]["enabled"] + assert len(study.get_preferences()) == 0 + + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference/{history_uuid}", + "PUT", + body=json.dumps( + { + "enable": True, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert histories[0]["enabled"] + preferences = study.get_preferences() + preferences.sort(key=lambda x: (x[0].number, x[1].number)) + assert len(preferences) == 2 + better, worse = preferences[0] + assert better.number == 0 + assert worse.number == 2 + better, worse = preferences[1] + assert better.number == 1 + assert worse.number == 2 + def test_create_study(self) -> None: for name, directions, expected_status in [ ("single-objective success", ["minimize"], 201), diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 85f25374..7b288968 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -2,10 +2,13 @@ from __future__ import annotations from typing import Callable +from optuna_dashboard._preferential_history import load_preference_history from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._preferential_history import switching_history from optuna_dashboard._serializer import serialize_preference_histories from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +from optuna_dashboard.preferential._system_attrs import get_preference from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier @@ -59,3 +62,58 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert len(preferences[i]) == 2 assert preferences[i][0] == best assert preferences[i][1] == worst + + +@parametrize_storages +def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + study.mark_comparison_ready(trial) + + study_id = study._study._study_id + + history_uuid = report_history( + study_id=study_id, + storage=storage, + input_data={ + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + }, + ) + switching_history(study_id, storage, history_uuid, False) + history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) + preference = get_preference(study_id, storage, history.preference_uuid) + assert history.mode == "ChooseWorst" + assert history.candidates == [0, 1, 2] + assert history.clicked == 1 + assert len(history.evacuated_preference) == 2 + assert len(preference) == 0 + + switching_history(study_id, storage, history_uuid, False) + history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) + preference = get_preference(study_id, storage, history.preference_uuid) + assert len(history.evacuated_preference) == 2 + assert len(preference) == 0 + + switching_history(study_id, storage, history_uuid, True) + history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) + preference = get_preference(study_id, storage, history.preference_uuid) + assert history.mode == "ChooseWorst" + assert history.candidates == [0, 1, 2] + assert history.clicked == 1 + assert len(history.evacuated_preference) == 0 + assert len(preference) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preference[i]) == 2 + assert preference[i][0] == best + assert preference[i][1] == worst + + switching_history(study_id, storage, history_uuid, True) + history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) + preference = get_preference(study_id, storage, history.preference_uuid) + assert len(history.evacuated_preference) == 0 + assert len(preference) == 2 From 6928e0c304e61648b9cba28f1b48ef47c63da99e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 15:32:35 +0900 Subject: [PATCH 013/104] add python api --- optuna_dashboard/_app.py | 23 ++ optuna_dashboard/_preference_setting.py | 60 +++++ optuna_dashboard/_serializer.py | 6 + optuna_dashboard/ts/action.ts | 21 ++ optuna_dashboard/ts/apiClient.ts | 20 ++ .../ts/components/PreferentialTrials.tsx | 212 ++++++++++-------- optuna_dashboard/ts/state.ts | 11 - optuna_dashboard/ts/types/index.d.ts | 3 + 8 files changed, 257 insertions(+), 99 deletions(-) create mode 100644 optuna_dashboard/_preference_setting.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1c439408..70acc13f 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -27,6 +27,7 @@ from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_output_component from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -284,6 +285,28 @@ def create_app( response.status = 204 return {} + @app.post("/api/studies//component") + @json_api_view + def post_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("component_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["Note", "Artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'Note' or 'Artifact'."} + + _register_output_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..93b1a34c --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" +_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" + + +def _register_output_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, + value=component_type, + ) + if artifact_key is not None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + value=artifact_key, + ) + + +def register_output_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str = "", +) -> None: + """Register output component to the study. + + Args: + study: + The study to register the output component. + component_type: + The type of the output component. + artifact_key: + When the component_type is "Artifact", + this argument is used as the attribute key of the artifact. + Each trial displays the artifact whose id is the value of the attribute. + """ + _register_output_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 0fa04124..b2494068 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -14,6 +14,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -155,6 +157,10 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: + serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] + if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: + serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] return serialized diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index 017751fa..f0c82e2b 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -16,6 +16,7 @@ import { deleteArtifactAPI, reportPreferenceAPI, skipPreferentialTrialAPI, + reportFeedbackComponentAPI, } from "./apiClient" import { graphVisibilityState, @@ -609,6 +610,25 @@ export const actionCreator = () => { }) } + const updateFeedbackComponent = ( + studyId: number, + compoennt_type: FeedbackComponentType, + artifact_key?: string + ) => { + reportFeedbackComponentAPI(studyId, compoennt_type, artifact_key).catch( + (err) => { + const reason = err.response?.data.reason + enqueueSnackbar( + `Failed to report feedback component. Reason: ${reason}`, + { + variant: "error", + } + ) + console.log(err) + } + ) + } + return { updateAPIMeta, updateStudyDetail, @@ -630,6 +650,7 @@ export const actionCreator = () => { saveTrialUserAttrs, updatePreference, skipPreferentialTrial, + updateFeedbackComponent, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 25ca3541..caa57716 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -70,6 +70,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + feedback_component_type?: string + feedback_artifact_key?: string } export const getStudyDetailAPI = ( @@ -105,6 +107,9 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + feedback_component_type: res.data + .feedback_component_type as FeedbackComponentType, + feedback_artifact_key: res.data.feedback_artifact_key, } }) } @@ -337,3 +342,18 @@ export const skipPreferentialTrialAPI = ( return }) } + +export const reportFeedbackComponentAPI = ( + studyId: number, + component_type: FeedbackComponentType, + artifact_key?: string +): Promise => { + return axiosInstance + .post(`/api/studies/${studyId}/component`, { + component_type: component_type, + artifact_key: artifact_key, + }) + .then(() => { + return + }) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 06bdad9d..a5d1e4af 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState } from "react" +import React, { FC, useEffect, useState } from "react" import { Typography, Box, @@ -11,7 +11,6 @@ import { Select, FormControl, FormLabel, - TextField, Modal, } from "@mui/material" import OpenInFullIcon from "@mui/icons-material/OpenInFull" @@ -20,14 +19,8 @@ import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" import SettingsIcon from "@mui/icons-material/Settings" import red from "@mui/material/colors/red" -import { useRecoilValue, useSetRecoilState } from "recoil" import { actionCreator } from "../action" import { MarkdownRenderer } from "./Note" -import { - feedbackComponent, - FeedbackComponentType, - feedbackArtifactKey, -} from "../state" import { TrialArtifactActions, TrialArtifactContent, @@ -37,13 +30,12 @@ import { const FeedbackContent: FC<{ trial: Trial artifact?: Artifact -}> = ({ trial, artifact }) => { - const componentId = useRecoilValue(feedbackComponent) - - if (componentId === "note") { + componentId: FeedbackComponentType +}> = ({ trial, artifact, componentId }) => { + if (componentId === "Note") { return } - if (componentId === "artifact") { + if (componentId === "Artifact") { if (artifact === undefined) { return null } @@ -63,11 +55,11 @@ const FeedbackContent: FC<{ const ModalPage: FC<{ children: React.ReactNode displayFlag: boolean - setDisplayFlag: (flag: boolean) => void -}> = ({ children, displayFlag, setDisplayFlag }) => { + onClose: () => void +}> = ({ children, displayFlag, onClose }) => { const theme = useTheme() return ( - setDisplayFlag(false)}> + a.key === artifactKey)?.value const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) @@ -135,7 +127,7 @@ const PreferentialTrial: FC<{ > Trial {trial.number} - {componentId === "artifact" && artifact !== undefined ? ( + {componentId === "Artifact" && artifact !== undefined ? ( ) : null} - {componentId === "artifact" && artifact !== undefined ? ( + {componentId === "Artifact" && artifact !== undefined ? ( - + - + { + setDetailShown(false) + }} + > true} @@ -248,6 +249,102 @@ const PreferentialTrial: FC<{ ) } +const SettingsPage: FC<{ + studyDetail: StudyDetail + settingShown: boolean + setSettingShown: (flag: boolean) => void +}> = ({ studyDetail, settingShown, setSettingShown }) => { + const theme = useTheme() + const actions = actionCreator() + const [outputComponent, setOutputComponent] = useState( + studyDetail?.feedback_component_type ?? "Note" + ) + const [outputArtifactKey, setOutputArtifactKey] = useState( + studyDetail?.feedback_artifact_key ?? "" + ) + useEffect(() => { + if (studyDetail.feedback_component_type !== undefined) { + setOutputComponent(studyDetail.feedback_component_type) + } + if (studyDetail.feedback_artifact_key !== undefined) { + setOutputArtifactKey(studyDetail.feedback_artifact_key) + } + }, [studyDetail.feedback_component_type, studyDetail.feedback_artifact_key]) + const onClose = () => { + setSettingShown(false) + actions.updateFeedbackComponent( + studyDetail.id, + outputComponent, + outputArtifactKey + ) + } + + return ( + + + Settings + + + + Output Component: + + + {outputComponent === "Artifact" ? ( + + + User Attribute Key Corresponding to Output Artifact Id: + + + + ) : null} + + + ) +} + type DisplayTrials = { numbers: number[] last_number: number @@ -265,10 +362,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), }) const [settingShown, setSettingShown] = useState(false) - const outputComponent = useRecoilValue(feedbackComponent) - const setOutputComponent = useSetRecoilState(feedbackComponent) - const outputArtifactKey = useRecoilValue(feedbackArtifactKey) - const setOutputArtifactKey = useSetRecoilState(feedbackArtifactKey) const new_trails = studyDetail.best_trials.filter( (t) => displayTrials.last_number < t.number && @@ -346,68 +439,11 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ /> ))} - - - Settings - - - Output Component: - - - {outputComponent === "artifact" ? ( - - - User Attribute Key Corresponding to Output Artifact Id: - - - - ) : null} - - - + ) } diff --git a/optuna_dashboard/ts/state.ts b/optuna_dashboard/ts/state.ts index 7464adb3..44301bd1 100644 --- a/optuna_dashboard/ts/state.ts +++ b/optuna_dashboard/ts/state.ts @@ -56,17 +56,6 @@ export const artifactIsAvailable = atom({ default: false, }) -export type FeedbackComponentType = "note" | "artifact" -export const feedbackComponent = atom({ - key: "feedbackComponent", - default: "note", -}) - -export const feedbackArtifactKey = atom({ - key: "feedbackArtifactKey", - default: "", -}) - export const useStudyDetailValue = (studyId: number): StudyDetail | null => { const studyDetails = useRecoilValue(studyDetailsState) return studyDetails[studyId] || null diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 1720cc6b..241ea310 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type FeedbackComponentType = "Note" | "Artifact" type FloatDistribution = { type: "FloatDistribution" @@ -197,6 +198,8 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + feedback_component_type?: FeedbackComponentType + feedback_artifact_key?: string } type StudyDetails = { From da38286b69d9a16784b5730e3b65416f49bb7722 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 18:59:17 +0900 Subject: [PATCH 014/104] add tests --- optuna_dashboard/ts/components/TrialList.tsx | 284 ++++--------------- python_tests/test_api.py | 63 ++++ python_tests/test_preference_setting.py | 20 ++ 3 files changed, 138 insertions(+), 229 deletions(-) create mode 100644 python_tests/test_preference_setting.py diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index f73a97e1..3a1fcee6 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -370,6 +370,7 @@ export const TrialArtifactContent: FC<{ display: "flex", justifyContent: "center", alignItems: "center", + height: height, }} > + - - {artifact.filename} - - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - - - - - - - ) - } + + + + + ) } const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { diff --git a/python_tests/test_api.py b/python_tests/test_api.py index ae50e29a..6828b83d 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -9,6 +9,7 @@ from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study from optuna_dashboard.preferential import create_study +from optuna_dashboard._preference_setting import register_output_component from .wsgi_client import send_request @@ -151,6 +152,68 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + register_output_component(study, "Note") + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Artifact" + assert study_detail["feedback_artifact_key"] == "image" + + def test_change_component_type_only(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage) + register_output_component(study, "Artifact", "audio") + for _ in range(3): + trial = study.ask() + study.mark_comparison_ready(trial) + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Note"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Note" + assert study_detail["feedback_artifact_key"] == "audio" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..05727a04 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,20 @@ +from __future__ import annotations +from unittest import TestCase + +import optuna + +from optuna_dashboard._preference_setting import ( + register_output_component, + _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, +) +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_output_component(study, "Artifact", "image_key") + system_attrs = study._study.system_attrs + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" From 5cbab3fa6791a3a6f8e013b760dfbbe3d72138af Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 4 Sep 2023 19:01:06 +0900 Subject: [PATCH 015/104] fix by lint --- python_tests/test_api.py | 2 +- python_tests/test_preference_setting.py | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 6828b83d..51017ac8 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,8 +8,8 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard.preferential import create_study from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard.preferential import create_study from .wsgi_client import send_request diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 05727a04..033a9092 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -1,13 +1,11 @@ from __future__ import annotations + from unittest import TestCase import optuna - -from optuna_dashboard._preference_setting import ( - register_output_component, - _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, - _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, -) +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from optuna_dashboard._preference_setting import register_output_component from optuna_dashboard.preferential._study import PreferentialStudy From bfb9afc898183cffb14b8bf3022bfce429fb9de3 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 5 Sep 2023 14:58:59 +0900 Subject: [PATCH 016/104] fixed Feedback screen --- .../ts/components/PreferentialTrials.tsx | 151 ++++++++++-------- optuna_dashboard/ts/components/TrialList.tsx | 45 +++--- 2 files changed, 107 insertions(+), 89 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index a5d1e4af..ec597a09 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -6,7 +6,7 @@ import { Card, CardContent, CardActions, - CardActionArea, + Button, MenuItem, Select, FormControl, @@ -31,7 +31,9 @@ const FeedbackContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType -}> = ({ trial, artifact, componentId }) => { + width: string + minHeight: string +}> = ({ trial, artifact, componentId, width, minHeight }) => { if (componentId === "Note") { return } @@ -43,8 +45,8 @@ const FeedbackContent: FC<{ ) } @@ -96,9 +98,10 @@ const PreferentialTrial: FC<{ }> = ({ trial, studyDetail, hideTrial }) => { const theme = useTheme() const action = actionCreator() - const trialWidth = 500 + const trialWidth = 400 const trialHeight = 300 const [detailShown, setDetailShown] = useState(false) + const [buttonHover, setButtonHover] = useState(false) const componentId = studyDetail.feedback_component_type ?? "Note" const artifactKey = studyDetail.feedback_artifact_key const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value @@ -115,6 +118,13 @@ const PreferentialTrial: FC<{ /> ) } + const onFeedback = () => { + hideTrial() + const best_trials = studyDetail.best_trials + .map((t) => t.number) + .filter((t) => t !== trial.number) + action.updatePreference(trial.study_id, best_trials, [trial.number]) + } return ( - - { - hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) - }} + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + { @@ -430,7 +443,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => ( trial.number === t)} studyDetail={studyDetail} hideTrial={() => { diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 3a1fcee6..bb70c119 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -459,13 +459,17 @@ const TrialArtifact: FC<{ artifact: Artifact width: string height: string - buttons_width: number }> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog] = useDeleteArtifactDialog() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() const theme = useTheme() - const is_3d_model = + const is3dModel = artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") - const actions_width = is_3d_model ? theme.spacing(12) : theme.spacing(8) + const canDelete = trial.state === "Running" || trial.state === "Waiting" + let actionsCount = 1 + if (canDelete) actionsCount += 1 + if (is3dModel) actionsCount += 1 + const actionsWidth = theme.spacing(actionsCount * 4) return ( @@ -495,29 +499,31 @@ const TrialArtifact: FC<{ p: theme.spacing(0.5, 0), flexGrow: 1, wordWrap: "break-word", - maxWidth: `calc(100% - ${actions_width})`, + maxWidth: `calc(100% - ${actionsWidth})`, }} > {artifact.filename} - {is_3d_model ? ( + {is3dModel ? ( ) : null} - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - + {canDelete ? ( + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + ) : null} + {renderDeleteArtifactDialog()} ) } @@ -536,7 +543,6 @@ const TrialArtifact: FC<{ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() const action = actionCreator() - const [, renderDeleteArtifactDialog] = useDeleteArtifactDialog() const [dragOver, setDragOver] = useState(false) const width = "200px" @@ -648,7 +654,6 @@ const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { ) : null} - {renderDeleteArtifactDialog()} ) } From 3f586020a444e701dd676032c53af767ca753715 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 11:15:06 +0900 Subject: [PATCH 017/104] refactor --- optuna_dashboard/ts/components/TrialList.tsx | 563 ++++++++----------- 1 file changed, 227 insertions(+), 336 deletions(-) diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 4aceb619..bb70c119 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -22,6 +22,7 @@ import { CardActionArea, Modal, } from "@mui/material" +import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -319,20 +320,230 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() +export const TrialArtifactContent: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { + if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + + + ) + } else { + return ( + + + + ) + } +} + +export const TrialArtifactActions: FC<{ + trial: Trial + artifact: Artifact + sx: SxProps +}> = ({ trial, artifact, sx }) => { + const [open3dModelViewer, setOpen3dModelViewer] = useState(false) + + if (artifact.mimetype.startsWith("image")) { + return null + } else if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + <> + { + setOpen3dModelViewer(true) + }} + > + + + { + setOpen3dModelViewer(false) + }} + > + + + + + + ) + } + return null +} + +const TrialArtifact: FC<{ + trial: Trial + artifact: Artifact + width: string + height: string +}> = ({ trial, artifact, width, height }) => { const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = useDeleteArtifactDialog() + const theme = useTheme() + const is3dModel = + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + const canDelete = trial.state === "Running" || trial.state === "Waiting" + let actionsCount = 1 + if (canDelete) actionsCount += 1 + if (is3dModel) actionsCount += 1 + const actionsWidth = theme.spacing(actionsCount * 4) + + return ( + + + + + {artifact.filename} + + {is3dModel ? ( + + ) : null} + {canDelete ? ( + { + openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) + }} + > + + + ) : null} + + + + + {renderDeleteArtifactDialog()} + + ) +} + +const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const action = actionCreator() const [dragOver, setDragOver] = useState(false) - const [open3dModelViewer, setOpen3dModelViewer] = useState<{ - [key: string]: boolean - }>({}) const width = "200px" const height = "150px" @@ -382,334 +593,15 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { Artifacts - {trial.artifacts.map((a) => { - if (a.mimetype.startsWith("image")) { - return ( - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = true - return obj - }) - }} - > - - - { - setOpen3dModelViewer(() => { - const obj = { ...open3dModelViewer } - obj[a.artifact_id] = false - return obj - }) - }} - > - - - - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } - })} + {trial.artifacts.map((a) => ( + + ))} {trial.state === "Running" || trial.state === "Waiting" ? ( = ({ trial }) => { ) : null} - {renderDeleteArtifactDialog()} ) } From b044ec9b6840ebd1f78b901fc03d82345dce8c7b Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 15:51:25 +0900 Subject: [PATCH 018/104] wip --- optuna_dashboard/_serializer.py | 2 + .../preferential/_system_attrs.py | 13 +- optuna_dashboard/ts/apiClient.ts | 2 + optuna_dashboard/ts/components/App.tsx | 9 + optuna_dashboard/ts/components/AppDrawer.tsx | 17 + .../ts/components/PreferentialGraph.tsx | 156 ++ .../ts/components/StudyDetail.tsx | 12 + optuna_dashboard/ts/types/index.d.ts | 4 +- package-lock.json | 1411 +++++++++++++++++ package.json | 3 + webpack.config.js | 134 +- 11 files changed, 1698 insertions(+), 65 deletions(-) create mode 100644 optuna_dashboard/ts/components/PreferentialGraph.tsx diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 06b53c42..5a35f618 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -18,6 +18,7 @@ from ._named_objectives import get_objective_names from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY +from .preferential._system_attrs import _get_preferences if TYPE_CHECKING: @@ -162,6 +163,7 @@ def serialize_study_detail( serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["preferences"] = _get_preferences(system_attrs) return serialized diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 4cb0e288..7952475c 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -35,12 +35,8 @@ def report_preferences( return preference_id -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def _get_preferences(system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue @@ -48,6 +44,13 @@ def get_preferences( return preferences +def get_preferences( + study_id: int, + storage: BaseStorage, +) -> list[tuple[int, int]]: + return _get_preferences(storage.get_study_system_attrs(study_id)) + + def report_skip( study_id: int, trial_id: int, diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index e23fc2ff..da5e6793 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -92,6 +92,7 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] } @@ -128,6 +129,7 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, + preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory ), diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 8adf8895..4772687d 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -87,6 +87,15 @@ export const App: FC = () => { /> } /> + + } + /> ({ width: drawerWidth, @@ -246,6 +248,21 @@ export const AppDrawer: FC<{ + {studyDetail?.is_preferential && ( + + + + + + + + + )} > = ({ data, isConnectable }) => { + const theme = useTheme() + const trial = data.trial + if (trial === undefined) { + return null + } + const noteBody = trial.note.body + const noteFC = useMemo(() => { + return + }, [noteBody]) + return ( + + + + {noteFC} + + + ) +} + +const nodeTypes: NodeTypes = { + note: GraphNode, +} + +const createNode = (x: number, y: number, trial: Trial): Node => { + return { + id: `${trial.number}`, + type: "note", + data: { + label: `Trial ${trial.number}`, + trial: trial, + }, + position: { + x: x * 500, + y: y * 400, + }, + style: { + width: nodeWidth, + height: nodeHeight, + padding: 0, + }, + } +} + +const defaultEdgeOptions: DefaultEdgeOptions = { + animated: true, +} + +export const PreferentialGraph: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if (studyDetail === null || !studyDetail.is_preferential) { + return null + } + const [nodes, setNodes] = useState([]) + + const onNodesChange: OnNodesChange = useCallback( + (changes) => setNodes((nds) => applyNodeChanges(changes, nds)), + [setNodes] + ) + useEffect(() => { + setNodes((prev) => { + const newNodes: Node[] = [] + studyDetail.best_trials.forEach((trial, i) => { + newNodes.push(createNode(i, 0, trial)) + }) + if (studyDetail.preference_history !== undefined) { + const histories = [...studyDetail.preference_history] + histories?.reverse().forEach((history, i) => { + const y = history.candidates.findIndex((c) => c === history.clicked) + newNodes.push( + createNode(y, i + 1, studyDetail.trials[history.clicked]) + ) + }) + } + return newNodes + }) + }, [studyDetail]) + + const edges: Edge[] = + studyDetail.preferences?.map((p) => { + return { + id: `e${p[0]}-${p[1]}`, + source: `${p[0]}`, + target: `${p[1]}`, + style: { stroke: "#fff" }, + } as Edge + }) ?? [] + + return ( + + ) +} diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index 15cb7cf8..6db6c2ed 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -32,6 +32,7 @@ import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" import { PreferenceHistory } from "./PreferenceHistory" import { PreferentialAnalytics } from "./PreferentialAnalytics" +import { PreferentialGraph } from "./PreferentialGraph" interface ParamTypes { studyId: string @@ -176,6 +177,17 @@ export const StudyDetail: FC<{ /> ) + } else if (page === "graph") { + content = ( + + + + ) } else if (page == "preferenceHistory") { content = } diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 646d64cf..b7d8288b 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -198,6 +198,7 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets + preferences?: [number, number][] preference_history?: PreferenceHistory[] } @@ -208,7 +209,6 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } - type PreferenceHistory = { id: string preference_id: string @@ -217,3 +217,5 @@ type PreferenceHistory = { feedback_mode: PreferenceFeedbackMode timestamp: Date } + +declare module "*.css" diff --git a/package-lock.json b/package-lock.json index fc9915bd..c2bda719 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,6 +25,7 @@ "react-markdown": "^8.0.4", "react-router-dom": "^6.11.0", "react-syntax-highlighter": "^15.5.0", + "reactflow": "^11.8.3", "recoil": "^0.7.7", "rehype-mathjax": "^4.0.2", "rehype-raw": "^6.1.1", @@ -44,12 +45,14 @@ "@typescript-eslint/eslint-plugin": "^4.26.1", "@typescript-eslint/parser": "^4.26.1", "compression-webpack-plugin": "^10.0.0", + "css-loader": "^6.8.1", "esbuild-loader": "^2.18.0", "eslint": "^7.28.0", "jest": "^29.2.1", "jest-canvas-mock": "^2.3.1", "jest-environment-jsdom": "^29.3.1", "prettier": "^2.5.1", + "style-loader": "^3.3.3", "ts-jest": "^29.0.3", "ts-loader": "^9.2.7", "typescript": "^4.6.2", @@ -3446,6 +3449,264 @@ "loose-envify": "^1.1.0" } }, + "node_modules/@reactflow/background": { + "version": "11.2.8", + "resolved": "https://registry.npmjs.org/@reactflow/background/-/background-11.2.8.tgz", + "integrity": "sha512-5o41N2LygiNC2/Pk8Ak2rIJjXbKHfQ23/Y9LFsnAlufqwdzFqKA8txExpsMoPVHHlbAdA/xpQaMuoChGPqmyDw==", + "dependencies": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/background/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@reactflow/controls": { + "version": "11.1.19", + "resolved": "https://registry.npmjs.org/@reactflow/controls/-/controls-11.1.19.tgz", + "integrity": "sha512-Vo0LFfAYjiSRMLEII/aeBo+1MT2a0Yc7iLVnkuRTLzChC0EX+A2Fa+JlzeOEYKxXlN4qcDxckRNGR7092v1HOQ==", + "dependencies": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/controls/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@reactflow/core": { + "version": "11.8.3", + "resolved": "https://registry.npmjs.org/@reactflow/core/-/core-11.8.3.tgz", + "integrity": "sha512-y6DN8Wy4V4KQBGHFqlj9zWRjLJU6CgdnVwWaEA/PdDg/YUkFBMpZnXqTs60czinoA2rAcvsz50syLTPsj5e+Wg==", + "dependencies": { + "@types/d3": "^7.4.0", + "@types/d3-drag": "^3.0.1", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/core/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@reactflow/minimap": { + "version": "11.6.3", + "resolved": "https://registry.npmjs.org/@reactflow/minimap/-/minimap-11.6.3.tgz", + "integrity": "sha512-PSA28dk09RnBHOA1zb45fjQXz3UozSJZmsIpgq49O3trfVFlSgRapxNdGsughWLs7/emg2M5jmi6Vc+ejcfjvQ==", + "dependencies": { + "@reactflow/core": "11.8.3", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/minimap/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@reactflow/node-resizer": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@reactflow/node-resizer/-/node-resizer-2.1.5.tgz", + "integrity": "sha512-z/hJlsptd2vTx13wKouqvN/Kln08qbkA+YTJLohc2aJ6rx3oGn9yX4E4IqNxhA7zNqYEdrnc1JTEA//ifh9z3w==", + "dependencies": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.4", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/node-resizer/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/@reactflow/node-toolbar": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@reactflow/node-toolbar/-/node-toolbar-1.2.7.tgz", + "integrity": "sha512-vs+Wg1tjy3SuD7eoeTqEtscBfE9RY+APqC28urVvftkrtsN7KlnoQjqDG6aE45jWP4z+8bvFizRWjAhxysNLkg==", + "dependencies": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, + "node_modules/@reactflow/node-toolbar/node_modules/zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "dependencies": { + "use-sync-external-store": "1.2.0" + }, + "engines": { + "node": ">=12.7.0" + }, + "peerDependencies": { + "@types/react": ">=16.8", + "immer": ">=9.0", + "react": ">=16.8" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "immer": { + "optional": true + }, + "react": { + "optional": true + } + } + }, "node_modules/@remix-run/router": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.6.0.tgz", @@ -3645,6 +3906,228 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/d3": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.0.tgz", + "integrity": "sha512-jIfNVK0ZlxcuRDKtRS/SypEyOQ6UHaFQBKv032X45VvxSJ6Yi5G9behy9h6tNTHTDGh5Vq+KbmBjUWLgY4meCA==", + "dependencies": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "node_modules/@types/d3-array": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.0.7.tgz", + "integrity": "sha512-4/Q0FckQ8TBjsB0VdGFemJOG8BLXUB2KKlL0VmZ+eOYeOnTb/wDRQqYWpBmQ6IlvWkXwkYiot+n9Px2aTJ7zGQ==" + }, + "node_modules/@types/d3-axis": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.3.tgz", + "integrity": "sha512-SE3x/pLO/+GIHH17mvs1uUVPkZ3bHquGzvZpPAh4yadRy71J93MJBpgK/xY8l9gT28yTN1g9v3HfGSFeBMmwZw==", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-brush": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.3.tgz", + "integrity": "sha512-MQ1/M/B5ifTScHSe5koNkhxn2mhUPqXjGuKjjVYckplAPjP9t2I2sZafb/YVHDwhoXWZoSav+Q726eIbN3qprA==", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-chord": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.3.tgz", + "integrity": "sha512-keuSRwO02c7PBV3JMWuctIfdeJrVFI7RpzouehvBWL4/GGUB3PBNg/9ZKPZAgJphzmS2v2+7vr7BGDQw1CAulw==" + }, + "node_modules/@types/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==" + }, + "node_modules/@types/d3-contour": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.3.tgz", + "integrity": "sha512-x7G/tdDZt4m09XZnG2SutbIuQqmkNYqR9uhDMdPlpJbcwepkEjEWG29euFcgVA1k6cn92CHdDL9Z+fOnxnbVQw==", + "dependencies": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-delaunay": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.1.tgz", + "integrity": "sha512-tLxQ2sfT0p6sxdG75c6f/ekqxjyYR0+LwPrsO1mbC9YDBzPJhs2HbJJRrn8Ez1DBoHRo2yx7YEATI+8V1nGMnQ==" + }, + "node_modules/@types/d3-dispatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.3.tgz", + "integrity": "sha512-Df7KW3Re7G6cIpIhQtqHin8yUxUHYAqiE41ffopbmU5+FifYUNV7RVyTg8rQdkEagg83m14QtS8InvNb95Zqug==" + }, + "node_modules/@types/d3-drag": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.3.tgz", + "integrity": "sha512-82AuQMpBQjuXeIX4tjCYfWjpm3g7aGCfx6dFlxX2JlRaiME/QWcHzBsINl7gbHCODA2anPYlL31/Trj/UnjK9A==", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-dsv": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.2.tgz", + "integrity": "sha512-DooW5AOkj4AGmseVvbwHvwM/Ltu0Ks0WrhG6r5FG9riHT5oUUTHz6xHsHqJSVU8ZmPkOqlUEY2obS5C9oCIi2g==" + }, + "node_modules/@types/d3-ease": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.0.tgz", + "integrity": "sha512-aMo4eaAOijJjA6uU+GIeW018dvy9+oH5Y2VPPzjjfxevvGQ/oRDs+tfYC9b50Q4BygRR8yE2QCLsrT0WtAVseA==" + }, + "node_modules/@types/d3-fetch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.3.tgz", + "integrity": "sha512-/EsDKRiQkby3Z/8/AiZq8bsuLDo/tYHnNIZkUpSeEHWV7fHUl6QFBjvMPbhkKGk9jZutzfOkGygCV7eR/MkcXA==", + "dependencies": { + "@types/d3-dsv": "*" + } + }, + "node_modules/@types/d3-force": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.5.tgz", + "integrity": "sha512-EGG+IWx93ESSXBwfh/5uPuR9Hp8M6o6qEGU7bBQslxCvrdUBQZha/EFpu/VMdLU4B0y4Oe4h175nSm7p9uqFug==" + }, + "node_modules/@types/d3-format": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.1.tgz", + "integrity": "sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==" + }, + "node_modules/@types/d3-geo": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.0.4.tgz", + "integrity": "sha512-kmUK8rVVIBPKJ1/v36bk2aSgwRj2N/ZkjDT+FkMT5pgedZoPlyhaG62J+9EgNIgUXE6IIL0b7bkLxCzhE6U4VQ==", + "dependencies": { + "@types/geojson": "*" + } + }, + "node_modules/@types/d3-hierarchy": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.3.tgz", + "integrity": "sha512-GpSK308Xj+HeLvogfEc7QsCOcIxkDwLhFYnOoohosEzOqv7/agxwvJER1v/kTC+CY1nfazR0F7gnHo7GE41/fw==" + }, + "node_modules/@types/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==", + "dependencies": { + "@types/d3-color": "*" + } + }, + "node_modules/@types/d3-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.0.tgz", + "integrity": "sha512-0g/A+mZXgFkQxN3HniRDbXMN79K3CdTpLsevj+PXiTcb2hVyvkZUBg37StmgCQkaD84cUJ4uaDAWq7UJOQy2Tg==" + }, + "node_modules/@types/d3-polygon": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.0.tgz", + "integrity": "sha512-D49z4DyzTKXM0sGKVqiTDTYr+DHg/uxsiWDAkNrwXYuiZVd9o9wXZIo+YsHkifOiyBkmSWlEngHCQme54/hnHw==" + }, + "node_modules/@types/d3-quadtree": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.2.tgz", + "integrity": "sha512-QNcK8Jguvc8lU+4OfeNx+qnVy7c0VrDJ+CCVFS9srBo2GL9Y18CnIxBdTF3v38flrGy5s1YggcoAiu6s4fLQIw==" + }, + "node_modules/@types/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-IIE6YTekGczpLYo/HehAy3JGF1ty7+usI97LqraNa8IiDur+L44d0VOjAvFQWJVdZOJHukUJw+ZdZBlgeUsHOQ==" + }, + "node_modules/@types/d3-scale": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.4.tgz", + "integrity": "sha512-eq1ZeTj0yr72L8MQk6N6heP603ubnywSDRfNpi5enouR112HzGLS6RIvExCzZTraFF4HdzNpJMwA/zGiMoHUUw==", + "dependencies": { + "@types/d3-time": "*" + } + }, + "node_modules/@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "node_modules/@types/d3-selection": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.6.tgz", + "integrity": "sha512-2ACr96USZVjXR9KMD9IWi1Epo4rSDKnUtYn6q2SPhYxykvXTw9vR77lkFNruXVg4i1tzQtBxeDMx0oNvJWbF1w==" + }, + "node_modules/@types/d3-shape": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.2.tgz", + "integrity": "sha512-NN4CXr3qeOUNyK5WasVUV8NCSAx/CRVcwcb0BuuS1PiTqwIm6ABi1SyasLZ/vsVCFDArF+W4QiGzSry1eKYQ7w==", + "dependencies": { + "@types/d3-path": "*" + } + }, + "node_modules/@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, + "node_modules/@types/d3-time-format": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.0.tgz", + "integrity": "sha512-yjfBUe6DJBsDin2BMIulhSHmr5qNR5Pxs17+oW4DoVPyVIXZ+m6bs7j1UVKP08Emv6jRmYrYqxYzO63mQxy1rw==" + }, + "node_modules/@types/d3-timer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.0.tgz", + "integrity": "sha512-HNB/9GHqu7Fo8AQiugyJbv6ZxYz58wef0esl4Mv828w1ZKpAshw/uFWVDUcIB9KKFeFKoxS3cHY07FFgtTRZ1g==" + }, + "node_modules/@types/d3-transition": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.4.tgz", + "integrity": "sha512-512a4uCOjUzsebydItSXsHrPeQblCVk8IKjqCUmrlvBWkkVh3donTTxmURDo1YPwIVDh5YVwCAO6gR4sgimCPQ==", + "dependencies": { + "@types/d3-selection": "*" + } + }, + "node_modules/@types/d3-zoom": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.4.tgz", + "integrity": "sha512-cqkuY1ah9ZQre2POqjSLcM8g40UVya/qwEUrNYP2/rCVljbmqKCVcv+ebvwhlI5azIbSEL7m+os6n+WlYA43aA==", + "dependencies": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, "node_modules/@types/debug": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", @@ -3684,6 +4167,11 @@ "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==", "dev": true }, + "node_modules/@types/geojson": { + "version": "7946.0.10", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz", + "integrity": "sha512-Nmh0K3iWQJzniTuPRcJn5hxXkfB1T1pgB89SBig5PlJQU5yocazeu4jATJlaA0GYFKWMqDdvYemoSnF2pXgLVA==" + }, "node_modules/@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -4967,6 +5455,11 @@ "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA==", "dev": true }, + "node_modules/classcat": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", + "integrity": "sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g==" + }, "node_modules/cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -5137,6 +5630,59 @@ "node": ">= 8" } }, + "node_modules/css-loader": { + "version": "6.8.1", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.8.1.tgz", + "integrity": "sha512-xDAXtEVGlD0gJ07iclwWVkLoZOpEvAWaSyf6W18S2pOC//K8+qUDIx8IIT3D+HjnmkJPQeesOPv5aiUaJsCM2g==", + "dev": true, + "dependencies": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.21", + "postcss-modules-extract-imports": "^3.0.0", + "postcss-modules-local-by-default": "^4.0.3", + "postcss-modules-scope": "^3.0.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.3.8" + }, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, + "node_modules/css-loader/node_modules/semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true, + "bin": { + "cssesc": "bin/cssesc" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/cssfontparser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/cssfontparser/-/cssfontparser-1.2.1.tgz", @@ -5169,6 +5715,102 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" }, + "node_modules/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "dependencies": { + "d3-color": "1 - 3" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==", + "engines": { + "node": ">=12" + } + }, + "node_modules/d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "dependencies": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + }, + "engines": { + "node": ">=12" + }, + "peerDependencies": { + "d3-selection": "2 - 3" + } + }, + "node_modules/d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "dependencies": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -7158,6 +7800,18 @@ "node": ">=0.10.0" } }, + "node_modules/icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, "node_modules/ignore": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", @@ -11466,6 +12120,24 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "node_modules/nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, "node_modules/natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -11832,6 +12504,112 @@ "resolved": "https://registry.npmjs.org/plotly.js-dist-min/-/plotly.js-dist-min-2.22.0.tgz", "integrity": "sha512-2b7w4CQI06px8HVpKpgZtfuoDjuCLA26VlgdnG71UDBrJvtCYvXb39H4ElNv+CA1bbD4S98KanpWPRqTqlxBZw==" }, + "node_modules/postcss": { + "version": "8.4.29", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.29.tgz", + "integrity": "sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "dependencies": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/postcss-modules-extract-imports": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz", + "integrity": "sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw==", + "dev": true, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-local-by-default": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.3.tgz", + "integrity": "sha512-2/u2zraspoACtrbFRnTijMiQtb4GW4BvatjaG/bCjYQo8kLTdevCUlwuBHx2sCnSyrI3x3qj4ZK1j5LQBgzmwA==", + "dev": true, + "dependencies": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.1.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-scope": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz", + "integrity": "sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg==", + "dev": true, + "dependencies": { + "postcss-selector-parser": "^6.0.4" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "dev": true, + "dependencies": { + "icss-utils": "^5.0.0" + }, + "engines": { + "node": "^10 || ^12 || >= 14" + }, + "peerDependencies": { + "postcss": "^8.1.0" + } + }, + "node_modules/postcss-selector-parser": { + "version": "6.0.13", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz", + "integrity": "sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ==", + "dev": true, + "dependencies": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, "node_modules/potpack": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz", @@ -12172,6 +12950,23 @@ "react-dom": ">=16.13" } }, + "node_modules/reactflow": { + "version": "11.8.3", + "resolved": "https://registry.npmjs.org/reactflow/-/reactflow-11.8.3.tgz", + "integrity": "sha512-wuVxJOFqi1vhA4WAEJLK0JWx2TsTiWpxTXTRp/wvpqKInQgQcB49I2QNyNYsKJCQ6jjXektS7H+LXoaVK/pG4A==", + "dependencies": { + "@reactflow/background": "11.2.8", + "@reactflow/controls": "11.1.19", + "@reactflow/core": "11.8.3", + "@reactflow/minimap": "11.6.3", + "@reactflow/node-resizer": "2.1.5", + "@reactflow/node-toolbar": "1.2.7" + }, + "peerDependencies": { + "react": ">=17", + "react-dom": ">=17" + } + }, "node_modules/rechoir": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.1.tgz", @@ -12890,6 +13685,15 @@ "node": ">=0.10.0" } }, + "node_modules/source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/source-map-support": { "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", @@ -13041,6 +13845,22 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/style-loader": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.3.tgz", + "integrity": "sha512-53BiGLXAcll9maCYtZi2RCQZKa8NQQai5C4horqKyRmHj9H7QmcUyucrH+4KW/gBQbXM2AsB0axoEcFZPlfPcw==", + "dev": true, + "engines": { + "node": ">= 12.13.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/webpack" + }, + "peerDependencies": { + "webpack": "^5.0.0" + } + }, "node_modules/style-to-object": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.3.0.tgz", @@ -13806,6 +14626,20 @@ "requires-port": "^1.0.0" } }, + "node_modules/use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "node_modules/utility-types": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.10.0.tgz", @@ -16834,6 +17668,138 @@ } } }, + "@reactflow/background": { + "version": "11.2.8", + "resolved": "https://registry.npmjs.org/@reactflow/background/-/background-11.2.8.tgz", + "integrity": "sha512-5o41N2LygiNC2/Pk8Ak2rIJjXbKHfQ23/Y9LFsnAlufqwdzFqKA8txExpsMoPVHHlbAdA/xpQaMuoChGPqmyDw==", + "requires": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, + "@reactflow/controls": { + "version": "11.1.19", + "resolved": "https://registry.npmjs.org/@reactflow/controls/-/controls-11.1.19.tgz", + "integrity": "sha512-Vo0LFfAYjiSRMLEII/aeBo+1MT2a0Yc7iLVnkuRTLzChC0EX+A2Fa+JlzeOEYKxXlN4qcDxckRNGR7092v1HOQ==", + "requires": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, + "@reactflow/core": { + "version": "11.8.3", + "resolved": "https://registry.npmjs.org/@reactflow/core/-/core-11.8.3.tgz", + "integrity": "sha512-y6DN8Wy4V4KQBGHFqlj9zWRjLJU6CgdnVwWaEA/PdDg/YUkFBMpZnXqTs60czinoA2rAcvsz50syLTPsj5e+Wg==", + "requires": { + "@types/d3": "^7.4.0", + "@types/d3-drag": "^3.0.1", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, + "@reactflow/minimap": { + "version": "11.6.3", + "resolved": "https://registry.npmjs.org/@reactflow/minimap/-/minimap-11.6.3.tgz", + "integrity": "sha512-PSA28dk09RnBHOA1zb45fjQXz3UozSJZmsIpgq49O3trfVFlSgRapxNdGsughWLs7/emg2M5jmi6Vc+ejcfjvQ==", + "requires": { + "@reactflow/core": "11.8.3", + "@types/d3-selection": "^3.0.3", + "@types/d3-zoom": "^3.0.1", + "classcat": "^5.0.3", + "d3-selection": "^3.0.0", + "d3-zoom": "^3.0.0", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, + "@reactflow/node-resizer": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@reactflow/node-resizer/-/node-resizer-2.1.5.tgz", + "integrity": "sha512-z/hJlsptd2vTx13wKouqvN/Kln08qbkA+YTJLohc2aJ6rx3oGn9yX4E4IqNxhA7zNqYEdrnc1JTEA//ifh9z3w==", + "requires": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.4", + "d3-drag": "^3.0.0", + "d3-selection": "^3.0.0", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, + "@reactflow/node-toolbar": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/@reactflow/node-toolbar/-/node-toolbar-1.2.7.tgz", + "integrity": "sha512-vs+Wg1tjy3SuD7eoeTqEtscBfE9RY+APqC28urVvftkrtsN7KlnoQjqDG6aE45jWP4z+8bvFizRWjAhxysNLkg==", + "requires": { + "@reactflow/core": "11.8.3", + "classcat": "^5.0.3", + "zustand": "^4.4.1" + }, + "dependencies": { + "zustand": { + "version": "4.4.1", + "resolved": "https://registry.npmjs.org/zustand/-/zustand-4.4.1.tgz", + "integrity": "sha512-QCPfstAS4EBiTQzlaGP1gmorkh/UL1Leaj2tdj+zZCZ/9bm0WS7sI2wnfD5lpOszFqWJ1DcPnGoY8RDL61uokw==", + "requires": { + "use-sync-external-store": "1.2.0" + } + } + } + }, "@remix-run/router": { "version": "1.6.0", "resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.6.0.tgz", @@ -16998,6 +17964,228 @@ "@babel/types": "^7.3.0" } }, + "@types/d3": { + "version": "7.4.0", + "resolved": "https://registry.npmjs.org/@types/d3/-/d3-7.4.0.tgz", + "integrity": "sha512-jIfNVK0ZlxcuRDKtRS/SypEyOQ6UHaFQBKv032X45VvxSJ6Yi5G9behy9h6tNTHTDGh5Vq+KbmBjUWLgY4meCA==", + "requires": { + "@types/d3-array": "*", + "@types/d3-axis": "*", + "@types/d3-brush": "*", + "@types/d3-chord": "*", + "@types/d3-color": "*", + "@types/d3-contour": "*", + "@types/d3-delaunay": "*", + "@types/d3-dispatch": "*", + "@types/d3-drag": "*", + "@types/d3-dsv": "*", + "@types/d3-ease": "*", + "@types/d3-fetch": "*", + "@types/d3-force": "*", + "@types/d3-format": "*", + "@types/d3-geo": "*", + "@types/d3-hierarchy": "*", + "@types/d3-interpolate": "*", + "@types/d3-path": "*", + "@types/d3-polygon": "*", + "@types/d3-quadtree": "*", + "@types/d3-random": "*", + "@types/d3-scale": "*", + "@types/d3-scale-chromatic": "*", + "@types/d3-selection": "*", + "@types/d3-shape": "*", + "@types/d3-time": "*", + "@types/d3-time-format": "*", + "@types/d3-timer": "*", + "@types/d3-transition": "*", + "@types/d3-zoom": "*" + } + }, + "@types/d3-array": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/@types/d3-array/-/d3-array-3.0.7.tgz", + "integrity": "sha512-4/Q0FckQ8TBjsB0VdGFemJOG8BLXUB2KKlL0VmZ+eOYeOnTb/wDRQqYWpBmQ6IlvWkXwkYiot+n9Px2aTJ7zGQ==" + }, + "@types/d3-axis": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-axis/-/d3-axis-3.0.3.tgz", + "integrity": "sha512-SE3x/pLO/+GIHH17mvs1uUVPkZ3bHquGzvZpPAh4yadRy71J93MJBpgK/xY8l9gT28yTN1g9v3HfGSFeBMmwZw==", + "requires": { + "@types/d3-selection": "*" + } + }, + "@types/d3-brush": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-brush/-/d3-brush-3.0.3.tgz", + "integrity": "sha512-MQ1/M/B5ifTScHSe5koNkhxn2mhUPqXjGuKjjVYckplAPjP9t2I2sZafb/YVHDwhoXWZoSav+Q726eIbN3qprA==", + "requires": { + "@types/d3-selection": "*" + } + }, + "@types/d3-chord": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-chord/-/d3-chord-3.0.3.tgz", + "integrity": "sha512-keuSRwO02c7PBV3JMWuctIfdeJrVFI7RpzouehvBWL4/GGUB3PBNg/9ZKPZAgJphzmS2v2+7vr7BGDQw1CAulw==" + }, + "@types/d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/@types/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-HKuicPHJuvPgCD+np6Se9MQvS6OCbJmOjGvylzMJRlDwUXjKTTXs6Pwgk79O09Vj/ho3u1ofXnhFOaEWWPrlwA==" + }, + "@types/d3-contour": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-contour/-/d3-contour-3.0.3.tgz", + "integrity": "sha512-x7G/tdDZt4m09XZnG2SutbIuQqmkNYqR9uhDMdPlpJbcwepkEjEWG29euFcgVA1k6cn92CHdDL9Z+fOnxnbVQw==", + "requires": { + "@types/d3-array": "*", + "@types/geojson": "*" + } + }, + "@types/d3-delaunay": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-delaunay/-/d3-delaunay-6.0.1.tgz", + "integrity": "sha512-tLxQ2sfT0p6sxdG75c6f/ekqxjyYR0+LwPrsO1mbC9YDBzPJhs2HbJJRrn8Ez1DBoHRo2yx7YEATI+8V1nGMnQ==" + }, + "@types/d3-dispatch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-dispatch/-/d3-dispatch-3.0.3.tgz", + "integrity": "sha512-Df7KW3Re7G6cIpIhQtqHin8yUxUHYAqiE41ffopbmU5+FifYUNV7RVyTg8rQdkEagg83m14QtS8InvNb95Zqug==" + }, + "@types/d3-drag": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-drag/-/d3-drag-3.0.3.tgz", + "integrity": "sha512-82AuQMpBQjuXeIX4tjCYfWjpm3g7aGCfx6dFlxX2JlRaiME/QWcHzBsINl7gbHCODA2anPYlL31/Trj/UnjK9A==", + "requires": { + "@types/d3-selection": "*" + } + }, + "@types/d3-dsv": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-dsv/-/d3-dsv-3.0.2.tgz", + "integrity": "sha512-DooW5AOkj4AGmseVvbwHvwM/Ltu0Ks0WrhG6r5FG9riHT5oUUTHz6xHsHqJSVU8ZmPkOqlUEY2obS5C9oCIi2g==" + }, + "@types/d3-ease": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-ease/-/d3-ease-3.0.0.tgz", + "integrity": "sha512-aMo4eaAOijJjA6uU+GIeW018dvy9+oH5Y2VPPzjjfxevvGQ/oRDs+tfYC9b50Q4BygRR8yE2QCLsrT0WtAVseA==" + }, + "@types/d3-fetch": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/d3-fetch/-/d3-fetch-3.0.3.tgz", + "integrity": "sha512-/EsDKRiQkby3Z/8/AiZq8bsuLDo/tYHnNIZkUpSeEHWV7fHUl6QFBjvMPbhkKGk9jZutzfOkGygCV7eR/MkcXA==", + "requires": { + "@types/d3-dsv": "*" + } + }, + "@types/d3-force": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/d3-force/-/d3-force-3.0.5.tgz", + "integrity": "sha512-EGG+IWx93ESSXBwfh/5uPuR9Hp8M6o6qEGU7bBQslxCvrdUBQZha/EFpu/VMdLU4B0y4Oe4h175nSm7p9uqFug==" + }, + "@types/d3-format": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-format/-/d3-format-3.0.1.tgz", + "integrity": "sha512-5KY70ifCCzorkLuIkDe0Z9YTf9RR2CjBX1iaJG+rgM/cPP+sO+q9YdQ9WdhQcgPj1EQiJ2/0+yUkkziTG6Lubg==" + }, + "@types/d3-geo": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-geo/-/d3-geo-3.0.4.tgz", + "integrity": "sha512-kmUK8rVVIBPKJ1/v36bk2aSgwRj2N/ZkjDT+FkMT5pgedZoPlyhaG62J+9EgNIgUXE6IIL0b7bkLxCzhE6U4VQ==", + "requires": { + "@types/geojson": "*" + } + }, + "@types/d3-hierarchy": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/@types/d3-hierarchy/-/d3-hierarchy-3.1.3.tgz", + "integrity": "sha512-GpSK308Xj+HeLvogfEc7QsCOcIxkDwLhFYnOoohosEzOqv7/agxwvJER1v/kTC+CY1nfazR0F7gnHo7GE41/fw==" + }, + "@types/d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-jx5leotSeac3jr0RePOH1KdR9rISG91QIE4Q2PYTu4OymLTZfA3SrnURSLzKH48HmXVUru50b8nje4E79oQSQw==", + "requires": { + "@types/d3-color": "*" + } + }, + "@types/d3-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-path/-/d3-path-3.0.0.tgz", + "integrity": "sha512-0g/A+mZXgFkQxN3HniRDbXMN79K3CdTpLsevj+PXiTcb2hVyvkZUBg37StmgCQkaD84cUJ4uaDAWq7UJOQy2Tg==" + }, + "@types/d3-polygon": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-polygon/-/d3-polygon-3.0.0.tgz", + "integrity": "sha512-D49z4DyzTKXM0sGKVqiTDTYr+DHg/uxsiWDAkNrwXYuiZVd9o9wXZIo+YsHkifOiyBkmSWlEngHCQme54/hnHw==" + }, + "@types/d3-quadtree": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@types/d3-quadtree/-/d3-quadtree-3.0.2.tgz", + "integrity": "sha512-QNcK8Jguvc8lU+4OfeNx+qnVy7c0VrDJ+CCVFS9srBo2GL9Y18CnIxBdTF3v38flrGy5s1YggcoAiu6s4fLQIw==" + }, + "@types/d3-random": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@types/d3-random/-/d3-random-3.0.1.tgz", + "integrity": "sha512-IIE6YTekGczpLYo/HehAy3JGF1ty7+usI97LqraNa8IiDur+L44d0VOjAvFQWJVdZOJHukUJw+ZdZBlgeUsHOQ==" + }, + "@types/d3-scale": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-scale/-/d3-scale-4.0.4.tgz", + "integrity": "sha512-eq1ZeTj0yr72L8MQk6N6heP603ubnywSDRfNpi5enouR112HzGLS6RIvExCzZTraFF4HdzNpJMwA/zGiMoHUUw==", + "requires": { + "@types/d3-time": "*" + } + }, + "@types/d3-scale-chromatic": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-scale-chromatic/-/d3-scale-chromatic-3.0.0.tgz", + "integrity": "sha512-dsoJGEIShosKVRBZB0Vo3C8nqSDqVGujJU6tPznsBJxNJNwMF8utmS83nvCBKQYPpjCzaaHcrf66iTRpZosLPw==" + }, + "@types/d3-selection": { + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@types/d3-selection/-/d3-selection-3.0.6.tgz", + "integrity": "sha512-2ACr96USZVjXR9KMD9IWi1Epo4rSDKnUtYn6q2SPhYxykvXTw9vR77lkFNruXVg4i1tzQtBxeDMx0oNvJWbF1w==" + }, + "@types/d3-shape": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@types/d3-shape/-/d3-shape-3.1.2.tgz", + "integrity": "sha512-NN4CXr3qeOUNyK5WasVUV8NCSAx/CRVcwcb0BuuS1PiTqwIm6ABi1SyasLZ/vsVCFDArF+W4QiGzSry1eKYQ7w==", + "requires": { + "@types/d3-path": "*" + } + }, + "@types/d3-time": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time/-/d3-time-3.0.0.tgz", + "integrity": "sha512-sZLCdHvBUcNby1cB6Fd3ZBrABbjz3v1Vm90nysCQ6Vt7vd6e/h9Lt7SiJUoEX0l4Dzc7P5llKyhqSi1ycSf1Hg==" + }, + "@types/d3-time-format": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-time-format/-/d3-time-format-4.0.0.tgz", + "integrity": "sha512-yjfBUe6DJBsDin2BMIulhSHmr5qNR5Pxs17+oW4DoVPyVIXZ+m6bs7j1UVKP08Emv6jRmYrYqxYzO63mQxy1rw==" + }, + "@types/d3-timer": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@types/d3-timer/-/d3-timer-3.0.0.tgz", + "integrity": "sha512-HNB/9GHqu7Fo8AQiugyJbv6ZxYz58wef0esl4Mv828w1ZKpAshw/uFWVDUcIB9KKFeFKoxS3cHY07FFgtTRZ1g==" + }, + "@types/d3-transition": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-transition/-/d3-transition-3.0.4.tgz", + "integrity": "sha512-512a4uCOjUzsebydItSXsHrPeQblCVk8IKjqCUmrlvBWkkVh3donTTxmURDo1YPwIVDh5YVwCAO6gR4sgimCPQ==", + "requires": { + "@types/d3-selection": "*" + } + }, + "@types/d3-zoom": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/@types/d3-zoom/-/d3-zoom-3.0.4.tgz", + "integrity": "sha512-cqkuY1ah9ZQre2POqjSLcM8g40UVya/qwEUrNYP2/rCVljbmqKCVcv+ebvwhlI5azIbSEL7m+os6n+WlYA43aA==", + "requires": { + "@types/d3-interpolate": "*", + "@types/d3-selection": "*" + } + }, "@types/debug": { "version": "4.1.7", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.7.tgz", @@ -17037,6 +18225,11 @@ "integrity": "sha512-CuPgU6f3eT/XgKKPqKd/gLZV1Xmvf1a2R5POBOGQa6uv82xpls89HU5zKeVoyR8XzHd1RGNOlQlvUe3CFkjWNQ==", "dev": true }, + "@types/geojson": { + "version": "7946.0.10", + "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.10.tgz", + "integrity": "sha512-Nmh0K3iWQJzniTuPRcJn5hxXkfB1T1pgB89SBig5PlJQU5yocazeu4jATJlaA0GYFKWMqDdvYemoSnF2pXgLVA==" + }, "@types/graceful-fs": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/graceful-fs/-/graceful-fs-4.1.5.tgz", @@ -18059,6 +19252,11 @@ "integrity": "sha512-cOU9usZw8/dXIXKtwa8pM0OTJQuJkxMN6w30csNRUerHfeQ5R6U3kkU/FtJeIf3M202OHfY2U8ccInBG7/xogA==", "dev": true }, + "classcat": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/classcat/-/classcat-5.0.4.tgz", + "integrity": "sha512-sbpkOw6z413p+HDGcBENe498WM9woqWHiJxCq7nvmxe9WmrUmqfAcxpIwAiMtM5Q3AhYkzXcNQHqsWq0mND51g==" + }, "cliui": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", @@ -18189,6 +19387,39 @@ "which": "^2.0.1" } }, + "css-loader": { + "version": "6.8.1", + "resolved": "https://registry.npmjs.org/css-loader/-/css-loader-6.8.1.tgz", + "integrity": "sha512-xDAXtEVGlD0gJ07iclwWVkLoZOpEvAWaSyf6W18S2pOC//K8+qUDIx8IIT3D+HjnmkJPQeesOPv5aiUaJsCM2g==", + "dev": true, + "requires": { + "icss-utils": "^5.1.0", + "postcss": "^8.4.21", + "postcss-modules-extract-imports": "^3.0.0", + "postcss-modules-local-by-default": "^4.0.3", + "postcss-modules-scope": "^3.0.0", + "postcss-modules-values": "^4.0.0", + "postcss-value-parser": "^4.2.0", + "semver": "^7.3.8" + }, + "dependencies": { + "semver": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", + "dev": true, + "requires": { + "lru-cache": "^6.0.0" + } + } + } + }, + "cssesc": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz", + "integrity": "sha512-/Tb/JcjK111nNScGob5MNtsntNM1aCNUDipB/TkwZFhyDrrE47SOx/18wF2bbjgc3ZzCSKW1T5nt5EbFoAz/Vg==", + "dev": true + }, "cssfontparser": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/cssfontparser/-/cssfontparser-1.2.1.tgz", @@ -18220,6 +19451,72 @@ "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.1.2.tgz", "integrity": "sha512-I7K1Uu0MBPzaFKg4nI5Q7Vs2t+3gWWW648spaF+Rg7pI9ds18Ugn+lvg4SHczUdKlHI5LWBXyqfS8+DufyBsgQ==" }, + "d3-color": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/d3-color/-/d3-color-3.1.0.tgz", + "integrity": "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==" + }, + "d3-dispatch": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-dispatch/-/d3-dispatch-3.0.1.tgz", + "integrity": "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==" + }, + "d3-drag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-drag/-/d3-drag-3.0.0.tgz", + "integrity": "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==", + "requires": { + "d3-dispatch": "1 - 3", + "d3-selection": "3" + } + }, + "d3-ease": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-ease/-/d3-ease-3.0.1.tgz", + "integrity": "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==" + }, + "d3-interpolate": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-interpolate/-/d3-interpolate-3.0.1.tgz", + "integrity": "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==", + "requires": { + "d3-color": "1 - 3" + } + }, + "d3-selection": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-selection/-/d3-selection-3.0.0.tgz", + "integrity": "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==" + }, + "d3-timer": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-timer/-/d3-timer-3.0.1.tgz", + "integrity": "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==" + }, + "d3-transition": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/d3-transition/-/d3-transition-3.0.1.tgz", + "integrity": "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==", + "requires": { + "d3-color": "1 - 3", + "d3-dispatch": "1 - 3", + "d3-ease": "1 - 3", + "d3-interpolate": "1 - 3", + "d3-timer": "1 - 3" + } + }, + "d3-zoom": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/d3-zoom/-/d3-zoom-3.0.0.tgz", + "integrity": "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==", + "requires": { + "d3-dispatch": "1 - 3", + "d3-drag": "2 - 3", + "d3-interpolate": "1 - 3", + "d3-selection": "2 - 3", + "d3-transition": "2 - 3" + } + }, "data-urls": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", @@ -19592,6 +20889,13 @@ "safer-buffer": ">= 2.1.2 < 3.0.0" } }, + "icss-utils": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/icss-utils/-/icss-utils-5.1.0.tgz", + "integrity": "sha512-soFhflCVWLfRNOPU3iv5Z9VUdT44xFRbzjLsEzSr5AQmgqPMTHdU3PMT1Cf1ssx8fLNJDA1juftYl+PUcv3MqA==", + "dev": true, + "requires": {} + }, "ignore": { "version": "5.2.1", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.2.1.tgz", @@ -22695,6 +23999,12 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "nanoid": { + "version": "3.3.6", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.6.tgz", + "integrity": "sha512-BGcqMMJuToF7i1rt+2PWSNVnWIkGCU78jBG3RxO/bZlnZPK2Cmi2QaffxGO/2RvWi9sL+FAiRiXMgsyxQ1DIDA==", + "dev": true + }, "natural-compare": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz", @@ -22956,6 +24266,69 @@ "resolved": "https://registry.npmjs.org/plotly.js-dist-min/-/plotly.js-dist-min-2.22.0.tgz", "integrity": "sha512-2b7w4CQI06px8HVpKpgZtfuoDjuCLA26VlgdnG71UDBrJvtCYvXb39H4ElNv+CA1bbD4S98KanpWPRqTqlxBZw==" }, + "postcss": { + "version": "8.4.29", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.29.tgz", + "integrity": "sha512-cbI+jaqIeu/VGqXEarWkRCCffhjgXc0qjBtXpqJhTBohMUjUQnbBr0xqX3vEKudc4iviTewcJo5ajcec5+wdJw==", + "dev": true, + "requires": { + "nanoid": "^3.3.6", + "picocolors": "^1.0.0", + "source-map-js": "^1.0.2" + } + }, + "postcss-modules-extract-imports": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-extract-imports/-/postcss-modules-extract-imports-3.0.0.tgz", + "integrity": "sha512-bdHleFnP3kZ4NYDhuGlVK+CMrQ/pqUm8bx/oGL93K6gVwiclvX5x0n76fYMKuIGKzlABOy13zsvqjb0f92TEXw==", + "dev": true, + "requires": {} + }, + "postcss-modules-local-by-default": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/postcss-modules-local-by-default/-/postcss-modules-local-by-default-4.0.3.tgz", + "integrity": "sha512-2/u2zraspoACtrbFRnTijMiQtb4GW4BvatjaG/bCjYQo8kLTdevCUlwuBHx2sCnSyrI3x3qj4ZK1j5LQBgzmwA==", + "dev": true, + "requires": { + "icss-utils": "^5.0.0", + "postcss-selector-parser": "^6.0.2", + "postcss-value-parser": "^4.1.0" + } + }, + "postcss-modules-scope": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-scope/-/postcss-modules-scope-3.0.0.tgz", + "integrity": "sha512-hncihwFA2yPath8oZ15PZqvWGkWf+XUfQgUGamS4LqoP1anQLOsOJw0vr7J7IwLpoY9fatA2qiGUGmuZL0Iqlg==", + "dev": true, + "requires": { + "postcss-selector-parser": "^6.0.4" + } + }, + "postcss-modules-values": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/postcss-modules-values/-/postcss-modules-values-4.0.0.tgz", + "integrity": "sha512-RDxHkAiEGI78gS2ofyvCsu7iycRv7oqw5xMWn9iMoR0N/7mf9D50ecQqUo5BZ9Zh2vH4bCUR/ktCqbB9m8vJjQ==", + "dev": true, + "requires": { + "icss-utils": "^5.0.0" + } + }, + "postcss-selector-parser": { + "version": "6.0.13", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.13.tgz", + "integrity": "sha512-EaV1Gl4mUEV4ddhDnv/xtj7sxwrwxdetHdWUGnT4VJQf+4d05v6lHYZr8N573k5Z0BViss7BDhfWtKS3+sfAqQ==", + "dev": true, + "requires": { + "cssesc": "^3.0.0", + "util-deprecate": "^1.0.2" + } + }, + "postcss-value-parser": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.2.0.tgz", + "integrity": "sha512-1NNCs6uurfkVbeXG4S8JFT9t19m45ICnif8zWLd5oPSZ50QnwMfK+H3jv408d4jw/7Bttv5axS5IiHoLaVNHeQ==", + "dev": true + }, "potpack": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/potpack/-/potpack-1.0.2.tgz", @@ -23200,6 +24573,19 @@ "debounce": "^1.2.1" } }, + "reactflow": { + "version": "11.8.3", + "resolved": "https://registry.npmjs.org/reactflow/-/reactflow-11.8.3.tgz", + "integrity": "sha512-wuVxJOFqi1vhA4WAEJLK0JWx2TsTiWpxTXTRp/wvpqKInQgQcB49I2QNyNYsKJCQ6jjXektS7H+LXoaVK/pG4A==", + "requires": { + "@reactflow/background": "11.2.8", + "@reactflow/controls": "11.1.19", + "@reactflow/core": "11.8.3", + "@reactflow/minimap": "11.6.3", + "@reactflow/node-resizer": "2.1.5", + "@reactflow/node-toolbar": "1.2.7" + } + }, "rechoir": { "version": "0.7.1", "resolved": "https://registry.npmjs.org/rechoir/-/rechoir-0.7.1.tgz", @@ -23720,6 +25106,12 @@ "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", "devOptional": true }, + "source-map-js": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", + "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", + "dev": true + }, "source-map-support": { "version": "0.5.21", "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.21.tgz", @@ -23838,6 +25230,13 @@ "integrity": "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==", "dev": true }, + "style-loader": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/style-loader/-/style-loader-3.3.3.tgz", + "integrity": "sha512-53BiGLXAcll9maCYtZi2RCQZKa8NQQai5C4horqKyRmHj9H7QmcUyucrH+4KW/gBQbXM2AsB0axoEcFZPlfPcw==", + "dev": true, + "requires": {} + }, "style-to-object": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/style-to-object/-/style-to-object-0.3.0.tgz", @@ -24360,6 +25759,18 @@ "requires-port": "^1.0.0" } }, + "use-sync-external-store": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", + "integrity": "sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA==", + "requires": {} + }, + "util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true + }, "utility-types": { "version": "3.10.0", "resolved": "https://registry.npmjs.org/utility-types/-/utility-types-3.10.0.tgz", diff --git a/package.json b/package.json index 82d587fe..acbb0622 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "react-markdown": "^8.0.4", "react-router-dom": "^6.11.0", "react-syntax-highlighter": "^15.5.0", + "reactflow": "^11.8.3", "recoil": "^0.7.7", "rehype-mathjax": "^4.0.2", "rehype-raw": "^6.1.1", @@ -53,12 +54,14 @@ "@typescript-eslint/eslint-plugin": "^4.26.1", "@typescript-eslint/parser": "^4.26.1", "compression-webpack-plugin": "^10.0.0", + "css-loader": "^6.8.1", "esbuild-loader": "^2.18.0", "eslint": "^7.28.0", "jest": "^29.2.1", "jest-canvas-mock": "^2.3.1", "jest-environment-jsdom": "^29.3.1", "prettier": "^2.5.1", + "style-loader": "^3.3.3", "ts-jest": "^29.0.3", "ts-loader": "^9.2.7", "typescript": "^4.6.2", diff --git a/webpack.config.js b/webpack.config.js index 001103bd..41abca1b 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -1,65 +1,81 @@ -const webpack = require('webpack'); +const webpack = require("webpack") -const mode = process.env.NODE_ENV === 'production' ? 'production' : 'development'; -const isDev = mode === 'development'; +const mode = + process.env.NODE_ENV === "production" ? "production" : "development" +const isDev = mode === "development" -const typeScriptLoader = process.env.TYPESCRIPT_LOADER === "esbuild-loader" ? { - test: /\.tsx?$/, - exclude: [/node_modules/], - loader: 'esbuild-loader', - options: { - loader: 'tsx', - tsconfigRaw: require('./tsconfig.json') - } -} : { - test: /\.tsx?$/, - exclude: [/node_modules/], - loader: 'ts-loader', - options: { - configFile: __dirname + '/tsconfig.json', - transpileOnly: isDev, - happyPackMode: true - } -} +const typeScriptLoader = + process.env.TYPESCRIPT_LOADER === "esbuild-loader" + ? { + test: /\.tsx?$/, + exclude: [/node_modules/], + loader: "esbuild-loader", + options: { + loader: "tsx", + tsconfigRaw: require("./tsconfig.json"), + }, + } + : { + test: /\.tsx?$/, + exclude: [/node_modules/], + loader: "ts-loader", + options: { + configFile: __dirname + "/tsconfig.json", + transpileOnly: isDev, + happyPackMode: true, + }, + } var config = { - mode, - entry: [__dirname + '/optuna_dashboard/ts/index.tsx'], - output: { - path: __dirname + '/optuna_dashboard/public/', - filename: 'bundle.js', - publicPath: '/public/' - }, - module: { - rules: [{oneOf: [typeScriptLoader]}] - }, - resolve: { - extensions: ['.ts', '.tsx', '.js'] - }, - plugins: [ - new webpack.DefinePlugin({ - 'APP_BAR_TITLE': JSON.stringify(process.env.APP_BAR_TITLE || "Optuna Dashboard"), - 'API_ENDPOINT': JSON.stringify(process.env.API_ENDPOINT), - 'URL_PREFIX': JSON.stringify(process.env.URL_PREFIX || "/dashboard") - }) - ] -}; - -if (isDev) { - config.devtool = 'source-map'; - config.cache = { - type: 'filesystem', - buildDependencies: { - config: [__filename], - } - } - console.log('= = = = = = = = = = = = = = = = = = ='); - console.log('DEVELOPMENT BUILD'); - console.log(process.env.TYPESCRIPT_LOADER === 'esbuild-loader' ? 'esbuild-loader' : 'ts-loader'); - console.log('= = = = = = = = = = = = = = = = = = ='); -} else { - const CompressionPlugin = require("compression-webpack-plugin"); - config.plugins.push(new CompressionPlugin()) + mode, + entry: [__dirname + "/optuna_dashboard/ts/index.tsx"], + output: { + path: __dirname + "/optuna_dashboard/public/", + filename: "bundle.js", + publicPath: "/public/", + }, + module: { + rules: [ + { oneOf: [typeScriptLoader] }, + { + test: /\.css$/, + use: ["style-loader", "css-loader"], + }, + ], + }, + resolve: { + extensions: [".ts", ".tsx", ".js"], + }, + plugins: [ + new webpack.DefinePlugin({ + APP_BAR_TITLE: JSON.stringify( + process.env.APP_BAR_TITLE || "Optuna Dashboard" + ), + API_ENDPOINT: JSON.stringify(process.env.API_ENDPOINT), + URL_PREFIX: JSON.stringify(process.env.URL_PREFIX || "/dashboard"), + }), + ], } -module.exports = config; \ No newline at end of file +if (isDev) { + config.devtool = "source-map" + config.cache = { + type: "filesystem", + buildDependencies: { + config: [__filename], + }, + } + console.log("= = = = = = = = = = = = = = = = = = =") + console.log("DEVELOPMENT BUILD") + console.log( + process.env.TYPESCRIPT_LOADER === "esbuild-loader" + ? "esbuild-loader" + : "ts-loader" + ) + console.log("= = = = = = = = = = = = = = = = = = =") +} else { + const CompressionPlugin = require("compression-webpack-plugin") + config.plugins.push(new CompressionPlugin()) +} + +module.exports = config From efb800ac0061b031b6357f1def27dbc7f213290b Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 6 Sep 2023 18:27:09 +0900 Subject: [PATCH 019/104] add graph page --- .../ts/components/PreferentialGraph.tsx | 186 +++++++++++++----- 1 file changed, 135 insertions(+), 51 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 48cc73a5..1b2b641c 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -1,44 +1,28 @@ import React, { FC, useState, useCallback, useMemo, useEffect } from "react" -import { - Box, - Card, - CardContent, - CardHeader, - Paper, - Typography, - useTheme, -} from "@mui/material" -import Grid2 from "@mui/material/Unstable_Grid2" -import { DataGrid, DataGridColumn } from "./DataGrid" -import { BestTrialsCard } from "./BestTrialsCard" -import { useStudyDetailValue, useStudySummaryValue } from "../state" -import { Contour } from "./GraphContour" +import { Card, CardContent, CardHeader, useTheme } from "@mui/material" import { MarkdownRenderer } from "./Note" import ReactFlow, { - addEdge, Node, NodeProps, NodeTypes, Edge, - FitViewOptions, DefaultEdgeOptions, applyNodeChanges, - applyEdgeChanges, OnNodesChange, - OnEdgesChange, - OnConnect, + MiniMap, Position, Handle, + XYPosition, } from "reactflow" import "reactflow/dist/style.css" const nodeWidth = 400 const nodeHeight = 300 +const nodeMargin = 50 type NodeData = { trial?: Trial } const GraphNode: FC> = ({ data, isConnectable }) => { - const theme = useTheme() const trial = data.trial if (trial === undefined) { return null @@ -77,7 +61,13 @@ const nodeTypes: NodeTypes = { note: GraphNode, } -const createNode = (x: number, y: number, trial: Trial): Node => { +const createNode = ( + x: number, + y: number, + trial: Trial, + bestGroupPos: XYPosition, + isBest: boolean +): Node => { return { id: `${trial.number}`, type: "note", @@ -86,62 +76,154 @@ const createNode = (x: number, y: number, trial: Trial): Node => { trial: trial, }, position: { - x: x * 500, - y: y * 400, + x: bestGroupPos.x + nodeMargin + x * (nodeWidth + nodeMargin), + y: bestGroupPos.y + nodeMargin + y * (nodeHeight + nodeMargin), }, style: { width: nodeWidth, height: nodeHeight, padding: 0, }, + parentNode: isBest ? "bestGroup" : undefined, } } +const updateNode = ( + addX: number, + addY: number, + node: Node, + trial: Trial, + isBest: boolean +): Node => { + return { + ...node, + position: { + x: node.position.x + addX * (nodeWidth + nodeMargin), + y: node.position.y + addY * (nodeHeight + nodeMargin), + }, + data: { + ...node.data, + trial: trial, + }, + parentNode: isBest ? "bestGroup" : undefined, + } +} + +const initNodes: Node[] = [ + { + id: "bestGroup", + type: "default", + position: { + x: 0, + y: 0, + }, + data: { + label: "Best Trials", + }, + style: { + width: 2 * nodeMargin, + height: nodeHeight + 2 * nodeMargin, + padding: 0, + backgroundColor: "rgb(255,0,0,0.1)", + }, + }, +] const defaultEdgeOptions: DefaultEdgeOptions = { animated: true, } -export const PreferentialGraph: FC<{ studyDetail: StudyDetail | null }> = ({ - studyDetail, -}) => { - if (studyDetail === null || !studyDetail.is_preferential) { - return null - } - const [nodes, setNodes] = useState([]) - +export const PreferentialGraph: FC<{ + studyDetail: StudyDetail | null +}> = ({ studyDetail }) => { + const theme = useTheme() + const [nodes, setNodes] = useState(initNodes) + const [edges, setEdges] = useState([]) + const [historyCount, setHistoryCount] = useState(0) const onNodesChange: OnNodesChange = useCallback( (changes) => setNodes((nds) => applyNodeChanges(changes, nds)), [setNodes] ) + const isDarkMode = theme.palette.mode === "dark" + useEffect(() => { + if (studyDetail === null) return + const newHistoryCount = + (studyDetail.preference_history?.length ?? 0) - historyCount + if (newHistoryCount === 0) return + setNodes((prev) => { const newNodes: Node[] = [] - studyDetail.best_trials.forEach((trial, i) => { - newNodes.push(createNode(i, 0, trial)) + const appendIds: string[] = studyDetail.best_trials.map((t) => + t.number.toString() + ) // 新しく追加する Node の id, これと "bestGroup" 以外は newHistoryCount だけ下にスライドする + studyDetail.preference_history?.slice(historyCount).forEach((history) => { + appendIds.push(history.clicked.toString()) }) - if (studyDetail.preference_history !== undefined) { - const histories = [...studyDetail.preference_history] - histories?.reverse().forEach((history, i) => { - const y = history.candidates.findIndex((c) => c === history.clicked) - newNodes.push( - createNode(y, i + 1, studyDetail.trials[history.clicked]) - ) + + const bestGroup = prev.find((node) => node.id === "bestGroup") + const bestGroupPos = bestGroup?.position ?? { x: 0, y: 0 } + console.log(bestGroupPos) + if (bestGroup !== undefined) { + newNodes.push({ + ...bestGroup, + style: { + ...bestGroup.style, + width: + nodeMargin + + studyDetail.best_trials.length * (nodeWidth + nodeMargin), + background: "rgb(255,200,200,0.1)", + }, }) } + + prev.forEach((node) => { + if (appendIds.includes(node.id)) return + if (node.id === "bestGroup") return + const trialNum = parseInt(node.id, 10) + if (node.id !== `${trialNum}`) { + console.error(`node.id is not trual number: ${node.id}`) + return + } + const trial = studyDetail.trials[trialNum] + newNodes.push(updateNode(0, newHistoryCount, node, trial, false)) + }) + const histories = studyDetail.preference_history?.slice(historyCount) + histories?.reverse().forEach((history, i) => { + const x = history.candidates.findIndex((c) => c === history.clicked) + newNodes.push( + createNode( + x, + i + 1, + studyDetail.trials[history.clicked], + bestGroupPos, + false + ) + ) + }) + studyDetail.best_trials.forEach((trial, i) => { + newNodes.push(createNode(i, 0, trial, bestGroupPos, true)) + }) return newNodes }) + setHistoryCount(studyDetail.preference_history?.length ?? 0) }, [studyDetail]) + useEffect(() => { + if (studyDetail?.preferences === undefined) return + setEdges( + studyDetail?.preferences?.map((p) => { + return { + id: `e${p[0]}-${p[1]}`, + source: `${p[0]}`, + target: `${p[1]}`, + style: { stroke: isDarkMode ? "#fff" : "#000" }, + } as Edge + }) ?? [] + ) + }, [studyDetail?.preferences, isDarkMode]) - const edges: Edge[] = - studyDetail.preferences?.map((p) => { - return { - id: `e${p[0]}-${p[1]}`, - source: `${p[0]}`, - target: `${p[1]}`, - style: { stroke: "#fff" }, - } as Edge - }) ?? [] - + if (studyDetail === null || !studyDetail.is_preferential) { + return null + } return ( = ({ nodeTypes={nodeTypes} zoomOnScroll={false} panOnScroll={true} - /> + > + + ) } From 862f012a6512cb9a367dd35c0537fcda3309ffaf Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 7 Sep 2023 17:07:24 +0900 Subject: [PATCH 020/104] add elkjs to position nodes automatically --- .../ts/components/PreferentialGraph.tsx | 304 ++++++++++-------- package-lock.json | 11 + package.json | 1 + 3 files changed, 180 insertions(+), 136 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 1b2b641c..defe8369 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -1,5 +1,12 @@ import React, { FC, useState, useCallback, useMemo, useEffect } from "react" -import { Card, CardContent, CardHeader, useTheme } from "@mui/material" +import { + Card, + CardContent, + useTheme, + Typography, + Box, + Chip, +} from "@mui/material" import { MarkdownRenderer } from "./Note" import ReactFlow, { Node, @@ -12,17 +19,22 @@ import ReactFlow, { MiniMap, Position, Handle, - XYPosition, } from "reactflow" import "reactflow/dist/style.css" +import ELK from "elkjs/lib/elk.bundled.js" +import { ElkNode } from "elkjs/lib/elk-api.js" +const elk = new ELK() const nodeWidth = 400 const nodeHeight = 300 -const nodeMargin = 50 +const nodeMargin = 60 + type NodeData = { trial?: Trial + isBest: boolean } const GraphNode: FC> = ({ data, isConnectable }) => { + const theme = useTheme() const trial = data.trial if (trial === undefined) { return null @@ -39,7 +51,25 @@ const GraphNode: FC> = ({ data, isConnectable }) => { overflow: "hidden", }} > - + + Trial {trial.number} + {data.isBest && ( + + )} + > = ({ data, isConnectable }) => { const nodeTypes: NodeTypes = { note: GraphNode, } - -const createNode = ( - x: number, - y: number, - trial: Trial, - bestGroupPos: XYPosition, - isBest: boolean -): Node => { - return { - id: `${trial.number}`, - type: "note", - data: { - label: `Trial ${trial.number}`, - trial: trial, - }, - position: { - x: bestGroupPos.x + nodeMargin + x * (nodeWidth + nodeMargin), - y: bestGroupPos.y + nodeMargin + y * (nodeHeight + nodeMargin), - }, - style: { - width: nodeWidth, - height: nodeHeight, - padding: 0, - }, - parentNode: isBest ? "bestGroup" : undefined, - } -} -const updateNode = ( - addX: number, - addY: number, - node: Node, - trial: Trial, - isBest: boolean -): Node => { - return { - ...node, - position: { - x: node.position.x + addX * (nodeWidth + nodeMargin), - y: node.position.y + addY * (nodeHeight + nodeMargin), - }, - data: { - ...node.data, - trial: trial, - }, - parentNode: isBest ? "bestGroup" : undefined, - } -} - -const initNodes: Node[] = [ - { - id: "bestGroup", - type: "default", - position: { - x: 0, - y: 0, - }, - data: { - label: "Best Trials", - }, - style: { - width: 2 * nodeMargin, - height: nodeHeight + 2 * nodeMargin, - padding: 0, - backgroundColor: "rgb(255,0,0,0.1)", - }, - }, -] - const defaultEdgeOptions: DefaultEdgeOptions = { animated: true, } +function reductionPreference( + input_preferences: [number, number][] +): [number, number][] { + const preferences: [number, number][] = [] + let n = 0 + for (const [source, target] of input_preferences) { + if ( + preferences.find((p) => p[0] === source && p[1] === target) !== undefined + ) { + continue + } + n = Math.max(n - 1, source, target) + 1 + preferences.push([source, target]) + } + if (n === 0) { + return [] + } + const graph: number[][] = Array.from({ length: n }, () => []) + const reverseGraph: number[][] = Array.from({ length: n }, () => []) + const degree: number[] = Array.from({ length: n }, () => 0) + for (const [source, target] of preferences) { + graph[source].push(target) + reverseGraph[target].push(source) + degree[target]++ + } + const topologicalOrder: number[] = [] + const q: number[] = [] + for (let i = 0; i < n; i++) { + if (degree[i] === 0) { + q.push(i) + } + } + while (q.length > 0) { + const v = q.pop() + if (v === undefined) break + topologicalOrder.push(v) + graph[v].forEach((u) => { + degree[u]-- + if (degree[u] === 0) { + q.push(u) + } + }) + } + if (topologicalOrder.length !== n) { + console.error("cycle detected") + return [] + } + + const response: [number, number][] = [] + const descendants: Set[] = Array.from( + { length: n }, + () => new Set() + ) + topologicalOrder.reverse().forEach((v) => { + const descendant = new Set([v]) + graph[v].forEach((u) => { + descendants[u].forEach((d) => descendant.add(d)) + }) + graph[v].forEach((u) => { + if (reverseGraph[u].filter((d) => descendant.has(d)).length === 1) { + response.push([v, u]) + } + }) + descendants[v] = descendant + }) + return response +} + export const PreferentialGraph: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail }) => { const theme = useTheme() - const [nodes, setNodes] = useState(initNodes) + const [nodes, setNodes] = useState([]) const [edges, setEdges] = useState([]) - const [historyCount, setHistoryCount] = useState(0) const onNodesChange: OnNodesChange = useCallback( (changes) => setNodes((nds) => applyNodeChanges(changes, nds)), [setNodes] @@ -147,70 +176,67 @@ export const PreferentialGraph: FC<{ useEffect(() => { if (studyDetail === null) return - const newHistoryCount = - (studyDetail.preference_history?.length ?? 0) - historyCount - if (newHistoryCount === 0) return - - setNodes((prev) => { - const newNodes: Node[] = [] - const appendIds: string[] = studyDetail.best_trials.map((t) => - t.number.toString() - ) // 新しく追加する Node の id, これと "bestGroup" 以外は newHistoryCount だけ下にスライドする - studyDetail.preference_history?.slice(historyCount).forEach((history) => { - appendIds.push(history.clicked.toString()) - }) - - const bestGroup = prev.find((node) => node.id === "bestGroup") - const bestGroupPos = bestGroup?.position ?? { x: 0, y: 0 } - console.log(bestGroupPos) - if (bestGroup !== undefined) { - newNodes.push({ - ...bestGroup, - style: { - ...bestGroup.style, - width: - nodeMargin + - studyDetail.best_trials.length * (nodeWidth + nodeMargin), - background: "rgb(255,200,200,0.1)", - }, - }) - } - - prev.forEach((node) => { - if (appendIds.includes(node.id)) return - if (node.id === "bestGroup") return - const trialNum = parseInt(node.id, 10) - if (node.id !== `${trialNum}`) { - console.error(`node.id is not trual number: ${node.id}`) - return - } - const trial = studyDetail.trials[trialNum] - newNodes.push(updateNode(0, newHistoryCount, node, trial, false)) - }) - const histories = studyDetail.preference_history?.slice(historyCount) - histories?.reverse().forEach((history, i) => { - const x = history.candidates.findIndex((c) => c === history.clicked) - newNodes.push( - createNode( - x, - i + 1, - studyDetail.trials[history.clicked], - bestGroupPos, - false - ) + if (!studyDetail.is_preferential || studyDetail.preferences === undefined) + return + const preferences = reductionPreference(studyDetail.preferences) + const graph: ElkNode = { + id: "root", + layoutOptions: { + "elk.algorithm": "layered", + "elk.direction": "DOWN", + "elk.layered.spacing.nodeNodeBetweenLayers": nodeMargin.toString(), + "elk.spacing.nodeNode": nodeMargin.toString(), + }, + children: studyDetail.trials.map((trial) => ({ + id: `${trial.number}`, + targetPosition: "top", + sourcePosition: "bottom", + width: nodeWidth, + height: nodeHeight, + })), + edges: preferences.map(([source, target]) => ({ + id: `e${source}-${target}`, + sources: [`${source}`], + targets: [`${target}`], + style: { stroke: isDarkMode ? "#fff" : "#000" }, + })), + } + elk + .layout(graph) + .then((layoutedGraph) => { + setNodes( + layoutedGraph.children?.map((node, index) => { + const trial = studyDetail.trials[index] + return { + id: `${trial.number}`, + type: "note", + data: { + label: `Trial ${trial.number}`, + trial: trial, + isBest: + studyDetail.best_trials.find( + (t) => t.number === trial.number + ) !== undefined, + }, + position: { + x: node.x ?? 0, + y: node.y ?? 0, + }, + style: { + width: nodeWidth, + height: nodeHeight, + padding: 0, + }, + deletable: false, + connectable: false, + draggable: false, + } + }) ?? [] ) }) - studyDetail.best_trials.forEach((trial, i) => { - newNodes.push(createNode(i, 0, trial, bestGroupPos, true)) - }) - return newNodes - }) - setHistoryCount(studyDetail.preference_history?.length ?? 0) - }, [studyDetail]) - useEffect(() => { - if (studyDetail?.preferences === undefined) return + .catch(console.error) setEdges( - studyDetail?.preferences?.map((p) => { + preferences.map((p) => { return { id: `e${p[0]}-${p[1]}`, source: `${p[0]}`, @@ -219,7 +245,7 @@ export const PreferentialGraph: FC<{ } as Edge }) ?? [] ) - }, [studyDetail?.preferences, isDarkMode]) + }, [studyDetail, isDarkMode]) if (studyDetail === null || !studyDetail.is_preferential) { return null @@ -233,6 +259,12 @@ export const PreferentialGraph: FC<{ nodeTypes={nodeTypes} zoomOnScroll={false} panOnScroll={true} + minZoom={0.1} + defaultViewport={{ + x: 0, + y: 0, + zoom: 0.5, + }} > diff --git a/package-lock.json b/package-lock.json index c2bda719..a3f13d65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "@react-three/fiber": "^8.13.6", "@types/three": "^0.154.0", "axios": "^1.2.1", + "elkjs": "^0.8.2", "notistack": "^3.0.1", "plotly.js-dist-min": "^2.22.0", "react": "^18.2.0", @@ -6035,6 +6036,11 @@ "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, + "node_modules/elkjs": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz", + "integrity": "sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ==" + }, "node_modules/emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", @@ -19690,6 +19696,11 @@ "integrity": "sha512-M8WEXFuKXMYMVr45fo8mq0wUrrJHheiKZf6BArTKk9ZBYCKJEOU5H8cdWgDT+qCVZf7Na4lVUaZsA+h6uA9+PA==", "dev": true }, + "elkjs": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/elkjs/-/elkjs-0.8.2.tgz", + "integrity": "sha512-L6uRgvZTH+4OF5NE/MBbzQx/WYpru1xCBE9respNj6qznEewGUIfhzmm7horWWxbNO2M0WckQypGctR8lH79xQ==" + }, "emittery": { "version": "0.13.1", "resolved": "https://registry.npmjs.org/emittery/-/emittery-0.13.1.tgz", diff --git a/package.json b/package.json index acbb0622..a675a1ce 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@react-three/fiber": "^8.13.6", "@types/three": "^0.154.0", "axios": "^1.2.1", + "elkjs": "^0.8.2", "notistack": "^3.0.1", "plotly.js-dist-min": "^2.22.0", "react": "^18.2.0", From 118a3e3dc4644e92b233af5e25796f32c74ef748 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 19:07:52 +0900 Subject: [PATCH 021/104] Refactor artifactUrlPath --- optuna_dashboard/ts/components/TrialList.tsx | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 84f98c0c..2ce525b6 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -384,6 +384,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { {trial.artifacts.map((a) => { + const artifactUrlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` if (a.mimetype.startsWith("image")) { return ( = ({ trial }) => { = ({ trial }) => { color="inherit" download={a.filename} sx={{ margin: "auto 0" }} - href={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`} + href={artifactUrlPath} > @@ -470,7 +471,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { }} > = ({ trial }) => { color="inherit" sx={{ margin: "auto 0" }} onClick={() => { - const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` - openThreejsArtifactModal(urlPath, a) + openThreejsArtifactModal(artifactUrlPath, a) }} > @@ -527,7 +527,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { color="inherit" sx={{ margin: "auto 0" }} download={a.filename} - href={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`} + href={artifactUrlPath} > @@ -557,7 +557,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { > @@ -599,7 +599,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { color="inherit" sx={{ margin: "auto 0" }} download={a.filename} - href={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`} + href={artifactUrlPath} > @@ -666,7 +666,7 @@ const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { color="inherit" sx={{ margin: "auto 0" }} download={a.filename} - href={`/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}`} + href={artifactUrlPath} > From b8995f2346d5df2db3c15323fa9aa10eeab8f3db Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 19:28:19 +0900 Subject: [PATCH 022/104] Split TrialArtifactCards component --- .../ts/components/TrialArtifactCards.tsx | 434 +++++++++++++++++ optuna_dashboard/ts/components/TrialList.tsx | 435 +----------------- 2 files changed, 437 insertions(+), 432 deletions(-) create mode 100644 optuna_dashboard/ts/components/TrialArtifactCards.tsx diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx new file mode 100644 index 00000000..7f9550e6 --- /dev/null +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -0,0 +1,434 @@ +import React, { + ChangeEventHandler, + DragEventHandler, + FC, + MouseEventHandler, + useRef, + useState, +} from "react" +import { + Typography, + Box, + useTheme, + IconButton, + Card, + CardContent, + CardMedia, + CardActionArea, +} from "@mui/material" +import UploadFileIcon from "@mui/icons-material/UploadFile" +import DownloadIcon from "@mui/icons-material/Download" +import DeleteIcon from "@mui/icons-material/Delete" +import FullscreenIcon from "@mui/icons-material/Fullscreen" +import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" + +import { actionCreator } from "../action" +import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" +import { + ThreejsArtifactViewer, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" + +export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const action = actionCreator() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() + const [dragOver, setDragOver] = useState(false) + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + + const width = "200px" + const height = "150px" + + const inputRef = useRef(null) + const handleClick: MouseEventHandler = () => { + if (!inputRef || !inputRef.current) { + return + } + inputRef.current.click() + } + const handleOnChange: ChangeEventHandler = (e) => { + const files = e.target.files + if (files === null) { + return + } + action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) + } + const handleDrop: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + const files = e.dataTransfer.files + setDragOver(false) + for (let i = 0; i < files.length; i++) { + action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) + } + } + const handleDragOver: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + const handleDragLeave: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + + return ( + <> + + Artifacts + + + {trial.artifacts.map((a) => { + const artifactUrlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` + if (a.mimetype.startsWith("image")) { + return ( + + + + + {a.filename} + + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + a + ) + }} + > + + + + + + + + ) + } else if ( + a.filename.endsWith(".stl") || + a.filename.endsWith(".3dm") + ) { + return ( + + + + + + + {a.filename} + + { + openThreejsArtifactModal(artifactUrlPath, a) + }} + > + + + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + a + ) + }} + > + + + + + + + + ) + } else if (a.mimetype.startsWith("audio")) { + return ( + + + + + + + {a.filename} + + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + a + ) + }} + > + + + + + + + + ) + } else { + return ( + + + + + + + {a.filename} + + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + a + ) + }} + > + + + + + + + + ) + } + })} + {trial.state === "Running" || trial.state === "Waiting" ? ( + + + + + + Upload a New File + + Drag your file here or click to browse. + + + + + ) : null} + + {renderDeleteArtifactDialog()} + {renderThreejsArtifactModal()} + + ) +} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index 2ce525b6..6922003a 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -1,13 +1,4 @@ -import React, { - ChangeEventHandler, - DragEventHandler, - FC, - MouseEventHandler, - ReactNode, - useMemo, - useRef, - useState, -} from "react" +import React, { FC, ReactNode, useMemo } from "react" import { Typography, Box, @@ -16,10 +7,6 @@ import { IconButton, Menu, MenuItem, - Card, - CardContent, - CardMedia, - CardActionArea, } from "@mui/material" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" @@ -31,11 +18,6 @@ import ListSubheader from "@mui/material/ListSubheader" import FilterListIcon from "@mui/icons-material/FilterList" import CheckBoxOutlineBlankIcon from "@mui/icons-material/CheckBoxOutlineBlank" import CheckBoxIcon from "@mui/icons-material/CheckBox" -import UploadFileIcon from "@mui/icons-material/UploadFile" -import DownloadIcon from "@mui/icons-material/Download" -import DeleteIcon from "@mui/icons-material/Delete" -import FullscreenIcon from "@mui/icons-material/Fullscreen" -import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import StopCircleIcon from "@mui/icons-material/StopCircle" import { TrialNote } from "./Note" @@ -44,12 +26,8 @@ import ListItemIcon from "@mui/material/ListItemIcon" import { useRecoilValue } from "recoil" import { artifactIsAvailable } from "../state" import { actionCreator } from "../action" -import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" import { TrialFormWidgets } from "./TrialFormWidgets" -import { - ThreejsArtifactViewer, - useThreejsArtifactModal, -} from "./ThreejsArtifactViewer" +import { TrialArtifactCards } from "./TrialArtifactCards" const states: TrialState[] = [ "Complete", @@ -321,418 +299,11 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -const TrialArtifact: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() - const [dragOver, setDragOver] = useState(false) - const [openThreejsArtifactModal, renderThreejsArtifactModal] = - useThreejsArtifactModal() - - const width = "200px" - const height = "150px" - - const inputRef = useRef(null) - const handleClick: MouseEventHandler = () => { - if (!inputRef || !inputRef.current) { - return - } - inputRef.current.click() - } - const handleOnChange: ChangeEventHandler = (e) => { - const files = e.target.files - if (files === null) { - return - } - action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) - } - const handleDrop: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - const files = e.dataTransfer.files - setDragOver(false) - for (let i = 0; i < files.length; i++) { - action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) - } - } - const handleDragOver: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(true) - } - const handleDragLeave: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(false) - } - - return ( - <> - - Artifacts - - - {trial.artifacts.map((a) => { - const artifactUrlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` - if (a.mimetype.startsWith("image")) { - return ( - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - - { - openThreejsArtifactModal(artifactUrlPath, a) - }} - > - - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } - })} - {trial.state === "Running" || trial.state === "Waiting" ? ( - - - - - - Upload a New File - - Drag your file here or click to browse. - - - - - ) : null} - - {renderDeleteArtifactDialog()} - {renderThreejsArtifactModal()} - - ) -} - const getTrialListLink = ( studyId: number, exclude: TrialState[], From 56866792476f00625c547c629b9c42a709d87545 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 19:32:24 +0900 Subject: [PATCH 023/104] Split TrialArtifactUploader --- .../ts/components/TrialArtifactCards.tsx | 188 +++++++++--------- 1 file changed, 99 insertions(+), 89 deletions(-) diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx index 7f9550e6..a6d2ffff 100644 --- a/optuna_dashboard/ts/components/TrialArtifactCards.tsx +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -31,52 +31,14 @@ import { export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() - const action = actionCreator() const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = useDeleteArtifactDialog() - const [dragOver, setDragOver] = useState(false) const [openThreejsArtifactModal, renderThreejsArtifactModal] = useThreejsArtifactModal() const width = "200px" const height = "150px" - const inputRef = useRef(null) - const handleClick: MouseEventHandler = () => { - if (!inputRef || !inputRef.current) { - return - } - inputRef.current.click() - } - const handleOnChange: ChangeEventHandler = (e) => { - const files = e.target.files - if (files === null) { - return - } - action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) - } - const handleDrop: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - const files = e.dataTransfer.files - setDragOver(false) - for (let i = 0; i < files.length; i++) { - action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) - } - } - const handleDragOver: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(true) - } - const handleDragLeave: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(false) - } - return ( <> = ({ trial }) => { ) } })} - {trial.state === "Running" || trial.state === "Waiting" ? ( - - - - - - Upload a New File - - Drag your file here or click to browse. - - - - - ) : null} + {renderDeleteArtifactDialog()} {renderThreejsArtifactModal()} ) } + +const TrialArtifactUploader: FC<{ + trial: Trial + width: string + height: string +}> = ({ trial, width, height }) => { + const theme = useTheme() + const action = actionCreator() + const [dragOver, setDragOver] = useState(false) + + if (trial.state !== "Running" && trial.state !== "Waiting") { + return null + } + const inputRef = useRef(null) + const handleClick: MouseEventHandler = () => { + if (!inputRef || !inputRef.current) { + return + } + inputRef.current.click() + } + const handleOnChange: ChangeEventHandler = (e) => { + const files = e.target.files + if (files === null) { + return + } + action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) + } + const handleDrop: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + const files = e.dataTransfer.files + setDragOver(false) + for (let i = 0; i < files.length; i++) { + action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) + } + } + const handleDragOver: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + const handleDragLeave: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + return ( + + + + + + Upload a New File + + Drag your file here or click to browse. + + + + + ) +} From 3d56318882bd8eba9ad2996568f05a737d7c6ec7 Mon Sep 17 00:00:00 2001 From: c-bata Date: Thu, 7 Sep 2023 20:00:29 +0900 Subject: [PATCH 024/104] Split TrialArtifactCardMedia component --- .../ts/components/ArtifactCardMedia.tsx | 41 +++ .../ts/components/TrialArtifactCards.tsx | 329 +++--------------- 2 files changed, 97 insertions(+), 273 deletions(-) create mode 100644 optuna_dashboard/ts/components/ArtifactCardMedia.tsx diff --git a/optuna_dashboard/ts/components/ArtifactCardMedia.tsx b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx new file mode 100644 index 00000000..6f577cf4 --- /dev/null +++ b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx @@ -0,0 +1,41 @@ +import React, { FC } from "react" +import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" +import { CardMedia } from "@mui/material" + +export const ArtifactCardMedia: FC<{ + artifact: Artifact + urlPath: string + height: string +}> = ({ artifact, urlPath, height }) => { + if ( + artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") + ) { + return ( + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + ) + } else if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } + return +} diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx index a6d2ffff..8191c898 100644 --- a/optuna_dashboard/ts/components/TrialArtifactCards.tsx +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -13,21 +13,17 @@ import { IconButton, Card, CardContent, - CardMedia, CardActionArea, } from "@mui/material" import UploadFileIcon from "@mui/icons-material/UploadFile" import DownloadIcon from "@mui/icons-material/Download" import DeleteIcon from "@mui/icons-material/Delete" import FullscreenIcon from "@mui/icons-material/Fullscreen" -import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import { actionCreator } from "../action" import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" -import { - ThreejsArtifactViewer, - useThreejsArtifactModal, -} from "./ThreejsArtifactViewer" +import { useThreejsArtifactModal } from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { const theme = useTheme() @@ -48,294 +44,81 @@ export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { Artifacts - {trial.artifacts.map((a) => { - const artifactUrlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${a.artifact_id}` - if (a.mimetype.startsWith("image")) { - return ( - { + const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${artifact.artifact_id}` + return ( + + + - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if ( - a.filename.endsWith(".stl") || - a.filename.endsWith(".3dm") - ) { - return ( - - - - - - - {a.filename} - + {artifact.filename} + + {artifact.filename.endsWith(".stl") || + artifact.filename.endsWith(".3dm") ? ( { - openThreejsArtifactModal(artifactUrlPath, a) + openThreejsArtifactModal(urlPath, artifact) }} > - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else if (a.mimetype.startsWith("audio")) { - return ( - - { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + artifact + ) }} > - - - + + - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } else { - return ( - - - - - - - {a.filename} - - { - openDeleteArtifactDialog( - trial.study_id, - trial.trial_id, - a - ) - }} - > - - - - - - - - ) - } + + + + + ) })} From cbb709d0df67ad1d0ff9dbae325ab0f15e386592 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 8 Sep 2023 13:57:35 +0900 Subject: [PATCH 025/104] Fix linter --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 226cbe3a..3a4675e5 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -318,7 +318,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): trials = study.get_trials(deepcopy=False) trials_with_preference = list({t for (b, w) in preferences for t in (b, w)}) ids = {t: i for i, t in enumerate(trials_with_preference)} - + trans = optuna._transform._SearchSpaceTransform( search_space, transform_log=True, transform_step=True, transform_0_1=True ) From ffd2e27002383b253072634737af66682b4588d7 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Fri, 8 Sep 2023 14:10:24 +0900 Subject: [PATCH 026/104] Update optuna_dashboard/preferential/samplers/gp.py Co-authored-by: Naoto Mizuno --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 3a4675e5..816a084f 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -25,7 +25,7 @@ def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: T assert cov_inv.shape == (dim, dim) sample_chain = initial_sample - conditional_std = 1 / torch.sqrt(torch.diag(cov_inv)) + conditional_std = torch.rsqrt(torch.diag(cov_inv)) scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] out = torch.empty((cycles + 1, dim), dtype=torch.float64) From 6039dd1f50abc1e6da86b4993d815946753a7f53 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Fri, 8 Sep 2023 14:10:32 +0900 Subject: [PATCH 027/104] Update optuna_dashboard/preferential/samplers/gp.py Co-authored-by: Naoto Mizuno --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 816a084f..4fafe98a 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -150,7 +150,7 @@ def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]: logz = torch.special.log_ndtr(-alpha) mean = 1 / (SQRT_HALF_PI * torch.special.erfcx(alpha * SQRT_HALF)) var = 1 - mean * (mean - alpha) - return (mean, var, logz) + return mean, var, logz def _orthants_MVN_EP( From 348d92e0125b11b235b775e8c0f35ff0cd57eae3 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Fri, 8 Sep 2023 14:10:47 +0900 Subject: [PATCH 028/104] Update optuna_dashboard/preferential/samplers/gp.py Co-authored-by: Naoto Mizuno --- optuna_dashboard/preferential/samplers/gp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 4fafe98a..2b7881c7 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -194,7 +194,7 @@ def _orthants_MVN_EP( mu = mu - Sxy * ((db + mean1 * da) * dr) cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :] log_zs[i] = logz - return (mu, cov, torch.sum(log_zs)) + return mu, cov, torch.sum(log_zs) _orthants_MVN_EP_jit = torch.jit.script(_orthants_MVN_EP) From 91edb0fc7cb8dcc1842f27b1b59724b98e4a9b8f Mon Sep 17 00:00:00 2001 From: Contramundum Date: Fri, 8 Sep 2023 14:11:21 +0900 Subject: [PATCH 029/104] Remove prints --- optuna_dashboard/preferential/samplers/gp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 3a4675e5..216dd3b7 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -248,8 +248,6 @@ class _PreferentialGP: def sample_gp(self, x: Tensor, preferences: Tensor) -> _SampledGP: self.fit_params_EP(x, preferences) - print({name: p.exp() for name, p in self.kernel.named_parameters()}) - print({"noise": self.log_noise.exp()}) with torch.no_grad(): cov_diff_diff_inv = _compute_cov_diff_diff_inv( From e1d81da70de4d6951995919fd3cae72e33c49501 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 8 Sep 2023 14:57:08 +0900 Subject: [PATCH 030/104] Fix width of ArtifactCard actions --- .../ts/components/ArtifactCardMedia.tsx | 10 +++++----- .../ts/components/ThreejsArtifactViewer.tsx | 6 ++++++ .../ts/components/TrialArtifactCards.tsx | 14 ++++++++++---- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/ts/components/ArtifactCardMedia.tsx b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx index 6f577cf4..a994aa41 100644 --- a/optuna_dashboard/ts/components/ArtifactCardMedia.tsx +++ b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx @@ -1,5 +1,8 @@ import React, { FC } from "react" -import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import { + ThreejsArtifactViewer, + isThreejsArtifact, +} from "./ThreejsArtifactViewer" import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import { CardMedia } from "@mui/material" @@ -8,10 +11,7 @@ export const ArtifactCardMedia: FC<{ urlPath: string height: string }> = ({ artifact, urlPath, height }) => { - if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { + if (isThreejsArtifact(artifact)) { return ( { + return ( + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + ) +} + interface ThreejsArtifactViewerProps { src: string width: string diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx index 8191c898..3e15f7ed 100644 --- a/optuna_dashboard/ts/components/TrialArtifactCards.tsx +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -22,7 +22,10 @@ import FullscreenIcon from "@mui/icons-material/Fullscreen" import { actionCreator } from "../action" import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" -import { useThreejsArtifactModal } from "./ThreejsArtifactViewer" +import { + useThreejsArtifactModal, + isThreejsArtifact, +} from "./ThreejsArtifactViewer" import { ArtifactCardMedia } from "./ArtifactCardMedia" export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { @@ -72,13 +75,16 @@ export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { p: theme.spacing(0.5, 0), flexGrow: 1, wordWrap: "break-word", - maxWidth: `calc(100% - ${theme.spacing(8)})`, + maxWidth: `calc(100% - ${ + isThreejsArtifact(artifact) + ? theme.spacing(12) + : theme.spacing(8) + })`, }} > {artifact.filename} - {artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") ? ( + {isThreejsArtifact(artifact) ? ( Date: Fri, 8 Sep 2023 15:23:05 +0900 Subject: [PATCH 031/104] fix tests --- optuna_dashboard/_preferential_history.py | 10 +- optuna_dashboard/_serializer.py | 2 +- .../preferential/_system_attrs.py | 2 +- .../ts/components/PreferenceHistory.tsx | 41 +++++-- python_tests/test_api.py | 19 ++- python_tests/test_preferential_history.py | 113 +++++++++--------- 6 files changed, 101 insertions(+), 86 deletions(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 3b9cd3e0..a8fe2148 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -8,7 +8,8 @@ import uuid from optuna.storages import BaseStorage -from .preferential._system_attrs import report_preferences, _SYSTEM_ATTR_PREFIX_PREFERENCE +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE +from .preferential._system_attrs import report_preferences _SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" @@ -43,7 +44,7 @@ def report_history( study_id: int, storage: BaseStorage, input_data: NewHistory, -) -> None: +) -> str: preferences = [] # TODO(moririn): Use TypeGuard after adding other history types. if input_data.mode == "ChooseWorst": @@ -78,14 +79,15 @@ def report_history( key=key, value=json.dumps(history), ) + return history_id def switching_history(study_id: int, storage: BaseStorage, uuid: str, enable: bool) -> None: system_attrs = storage.get_study_system_attrs(study_id) - history: History = system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, None) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, "")) if enable: preferences = [ - (best, history["clickedx"]) + (best, history["clicked"]) for best in history["candidates"] if best != history["clicked"] ] diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index cc9401b7..bcbe1ddf 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -187,7 +187,7 @@ def serialize_preference_history( "timestamp": choice["timestamp"], "candidates": choice["candidates"], "clicked": choice["clicked"], - "enabled": is_preference_valid(choice["preference_id"]), + "enabled": is_preference_valid(system_attrs, choice["preference_id"]), } histories.append(history) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 0c49a8a8..5ce83742 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -47,7 +47,7 @@ def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]] def is_preference_valid(study_system_attrs: dict[str, Any], uuid: str) -> bool: key = _SYSTEM_ATTR_PREFIX_PREFERENCE + uuid preference = study_system_attrs.get(key, []) - return len(preference) == 0 + return len(preference) > 0 def report_skip( diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index c9444237..bfacfc62 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -9,9 +9,9 @@ import { } from "@mui/material" import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" -import UndoIcon from "@mui/icons-material/Undo" -import RedoIcon from "@mui/icons-material/Redo" import OpenInFullIcon from "@mui/icons-material/OpenInFull" +import RestoreFromTrashIcon from "@mui/icons-material/RestoreFromTrash" +import DeleteIcon from "@mui/icons-material/Delete" import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" @@ -142,14 +142,13 @@ const ChoiceTrials: FC<{ trials: Trial[] study_id: number }> = ({ choice, trials, study_id }) => { + const [enabled, setEnabled] = useState(choice.enabled) const theme = useTheme() const worst_trials = new Set([choice.clicked]) const actions = actionCreator() - const handleUndo = () => { - actions.switchPreferentialHistory(study_id, choice.id, false) - } - const handleRedo = () => { - actions.switchPreferentialHistory(study_id, choice.id, true) + const handleSwitch = () => { + setEnabled(!enabled) + actions.switchPreferentialHistory(study_id, choice.id, !enabled) } return ( @@ -158,14 +157,32 @@ const ChoiceTrials: FC<{ marginBottom: theme.spacing(4), }} > - - {formatDate(choice.timestamp)} - + + {formatDate(choice.timestamp)} + + + {enabled ? : } + + None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -227,14 +226,14 @@ class APITestCase(TestCase): content_type="application/json", ) self.assertEqual(status, 204) - histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 assert histories[0]["enabled"] - history_uuid = histories[0]["uuid"] + history_id = histories[0]["id"] status, _, _ = send_request( app, - f"/api/studies/{study_id}/preference/{history_uuid}", + f"/api/studies/{study_id}/preference/{history_id}", "PUT", body=json.dumps( { @@ -244,14 +243,14 @@ class APITestCase(TestCase): content_type="application/json", ) self.assertEqual(status, 204) - histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 assert not histories[0]["enabled"] assert len(study.get_preferences()) == 0 status, _, _ = send_request( app, - f"/api/studies/{study_id}/preference/{history_uuid}", + f"/api/studies/{study_id}/preference/{history_id}", "PUT", body=json.dumps( { @@ -261,7 +260,7 @@ class APITestCase(TestCase): content_type="application/json", ) self.assertEqual(status, 204) - histories = serialize_preference_histories(storage.get_study_system_attrs(study_id)) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 assert histories[0]["enabled"] preferences = study.get_preferences() diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index eb5ffbce..e1e18b6e 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -1,9 +1,13 @@ from __future__ import annotations +import json from typing import Callable +from typing import TYPE_CHECKING +from optuna_dashboard._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._preferential_history import switching_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -12,6 +16,10 @@ from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier +if TYPE_CHECKING: + from optuna_dashboard._preferential_history import History + + @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: @@ -25,20 +33,12 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) report_history( study_id=study_id, storage=storage, - input_data=NewHistory( - mode="ChooseWorst", - candidates=[0, 1, 2], - clicked=1, - ), + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), ) report_history( study_id=study_id, storage=storage, - input_data=NewHistory( - mode="ChooseWorst", - candidates=[0, 2, 3, 4], - clicked=0, - ), + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 2, 3, 4], clicked=0), ) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) @@ -61,56 +61,53 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst -# TODO(moririn): Add tests for switching_history. -# @parametrize_storages -# def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: -# with storage_supplier() as storage: -# study = create_study(storage=storage, n_generate=5) -# for _ in range(5): -# trial = study.ask() -# trial.suggest_float("x", 0, 1) +@parametrize_storages +def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) -# study_id = study._study._study_id + study_id = study._study._study_id -# history_uuid = report_history( -# study_id=study_id, -# storage=storage, -# input_data={ -# "mode": "ChooseWorst", -# "candidates": [0, 1, 2], -# "clicked": 1, -# }, -# ) -# switching_history(study_id, storage, history_uuid, False) -# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) -# preference = get_preference(study_id, storage, history.preference_uuid) -# assert history.mode == "ChooseWorst" -# assert history.candidates == [0, 1, 2] -# assert history.clicked == 1 -# assert len(history.evacuated_preference) == 2 -# assert len(preference) == 0 + def get_preferences_history(history_id: str): + system_attrs = storage.get_study_system_attrs(study_id) + history: History = json.loads( + system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + history_id, "") + ) + preference: list[tuple[int, int]] = system_attrs.get( + _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] + ) + return preference, history -# switching_history(study_id, storage, history_uuid, False) -# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) -# preference = get_preference(study_id, storage, history.preference_uuid) -# assert len(history.evacuated_preference) == 2 -# assert len(preference) == 0 + history_id = report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), + ) + switching_history(study_id, storage, history_id, False) + preference, history = get_preferences_history(history_id) + assert history["mode"] == "ChooseWorst" + assert history["candidates"] == [0, 1, 2] + assert history["clicked"] == 1 + assert len(preference) == 0 -# switching_history(study_id, storage, history_uuid, True) -# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) -# preference = get_preference(study_id, storage, history.preference_uuid) -# assert history.mode == "ChooseWorst" -# assert history.candidates == [0, 1, 2] -# assert history.clicked == 1 -# assert len(history.evacuated_preference) == 0 -# assert len(preference) == 2 -# for i, (best, worst) in enumerate([(0, 1), (2, 1)]): -# assert len(preference[i]) == 2 -# assert preference[i][0] == best -# assert preference[i][1] == worst + switching_history(study_id, storage, history_id, False) + preference, history = get_preferences_history(history_id) + assert len(preference) == 0 -# switching_history(study_id, storage, history_uuid, True) -# history = load_preference_history(history_uuid, storage.get_study_system_attrs(study_id)) -# preference = get_preference(study_id, storage, history.preference_uuid) -# assert len(history.evacuated_preference) == 0 -# assert len(preference) == 2 + switching_history(study_id, storage, history_id, True) + preference, history = get_preferences_history(history_id) + assert history["mode"] == "ChooseWorst" + assert history["candidates"] == [0, 1, 2] + assert history["clicked"] == 1 + assert len(preference) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preference[i]) == 2 + assert preference[i][0] == best + assert preference[i][1] == worst + + switching_history(study_id, storage, history_id, True) + preference, history = get_preferences_history(history_id) + assert len(preference) == 2 From 2f1cb918862daccbcf2b7b835dfe874f4283f7f8 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 8 Sep 2023 16:57:11 +0900 Subject: [PATCH 032/104] add undo button on feedback screen --- .../ts/components/PreferenceHistory.tsx | 8 +-- .../ts/components/PreferentialTrials.tsx | 49 +++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index bfacfc62..0535dc2e 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -145,16 +145,17 @@ const ChoiceTrials: FC<{ const [enabled, setEnabled] = useState(choice.enabled) const theme = useTheme() const worst_trials = new Set([choice.clicked]) - const actions = actionCreator() + const action = actionCreator() const handleSwitch = () => { setEnabled(!enabled) - actions.switchPreferentialHistory(study_id, choice.id, !enabled) + action.switchPreferentialHistory(study_id, choice.id, !enabled) } return ( - {enabled ? : } + {choice.enabled ? : } {choice.candidates.map((trial_num, index) => ( diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 08efd9a7..ab7f04ea 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -14,6 +14,7 @@ import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" +import UndoIcon from "@mui/icons-material/Undo" import { actionCreator } from "../action" import { TrialListDetail } from "./TrialList" @@ -180,10 +181,13 @@ type DisplayTrials = { export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { + const [undoHistoryId, setUndoHistoryId] = useState(null) + if (studyDetail === null || !studyDetail.is_preferential) { return null } const theme = useTheme() + const action = actionCreator() const runningTrials = studyDetail.trials.filter((t) => t.state === "Running") const activeTrials = runningTrials.concat(studyDetail.best_trials) @@ -229,18 +233,45 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ } }) } + const latestHistoryId = studyDetail?.preference_history + ?.filter((h) => h.enabled) + .pop()?.id + if (undoHistoryId !== null && undoHistoryId !== latestHistoryId) { + setUndoHistoryId(null) + } return ( - - Which trial is the worst? - + + + Which trial is the worst? + + { + if (latestHistoryId === undefined) { + return + } + setUndoHistoryId(latestHistoryId) + action.switchPreferentialHistory( + studyDetail.id, + latestHistoryId, + false + ) + }} + sx={{ + margin: "auto 0 auto auto", + }} + > + + + {displayTrials.numbers.map((t, index) => ( Date: Fri, 8 Sep 2023 17:03:45 +0900 Subject: [PATCH 033/104] fix by lint --- optuna_dashboard/_serializer.py | 6 +++--- python_tests/test_preferential_history.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index bcbe1ddf..5ac9c55b 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -173,14 +173,14 @@ def serialize_study_detail( def serialize_preference_history( system_attrs: dict[str, Any], -) -> list[History]: - histories: list[History] = [] +) -> list[dict[str, Any]]: + histories: list[dict[str, Any]] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue choice: dict[str, Any] = json.loads(v) if choice["mode"] == "ChooseWorst": - history: ChooseWorstHistory = { + history = { "mode": "ChooseWorst", "id": choice["id"], "preference_id": choice["preference_id"], diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index e1e18b6e..75e622db 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -71,7 +71,7 @@ def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> N study_id = study._study._study_id - def get_preferences_history(history_id: str): + def get_preferences_history(history_id: str) -> tuple[list[tuple[int, int]], History]: system_attrs = storage.get_study_system_attrs(study_id) history: History = json.loads( system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + history_id, "") From 23db3e7c4ecd4bade3851bee474ffe1991dd556b Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 8 Sep 2023 17:09:43 +0900 Subject: [PATCH 034/104] fix by lint --- optuna_dashboard/_serializer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 5ac9c55b..35b9d6a7 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -25,9 +25,6 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict - from ._preferential_history import ChooseWorstHistory - from ._preferential_history import History - Attribute = TypedDict( "Attribute", { From d1347711cfbaac8bc257f5d4127654a9637bddd2 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 8 Sep 2023 17:50:46 +0900 Subject: [PATCH 035/104] Minor fixes on human-in-the-loop tutorial --- docs/tutorials/hitl.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 6d31ae55..6b0cf7c7 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -5,7 +5,7 @@ Tutorial: Human-in-the-loop Optimization In tasks involving image generation, natural language, or speech synthesis, evaluating results mechanically can be tough, and human evaluation becomes crucial. Until now, managing such tasks with Optuna has been challenging. However, the introduction of Optuna Dashboard enables humans and optimization algorithms to work interactively and execute the optimization process. -In this tutorial, we will explain how to optimize hyperparameters to generate a simple image using Optuna Dashboard. While the tutorial focuses on a simple task, the same approach can be applied to for instance optimize more complex images, natural language, and speech. +In this tutorial, we will explain how to optimize hyperparameters to generate a simple image using Optuna Dashboard. While the tutorial focuses on a simple task, the same approach can be applied to for instance optimize more complex images, natural language, and speech. The tutorial is organized as follows: @@ -93,7 +93,7 @@ Given the above system, we carry out HITL optimization as follows: Environment setup ^^^^^^^^^^^^^^^^^ -To run `the script `_ used in this tutorial, you need to install two libraries: +To run `the script `_ used in this tutorial, you need to install following libraries: .. code-block:: console From 59c6060be3d25239ba073f30b6615245c21dabc2 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 8 Sep 2023 13:58:53 +0900 Subject: [PATCH 036/104] Add tutorial for preferential optimization --- docs/tutorials/hitl.rst | 6 +- .../images/preferential-optimization/anim.gif | Bin 0 -> 271080 bytes .../system-architecture.png | Bin 0 -> 82341 bytes docs/tutorials/index.rst | 1 + docs/tutorials/preferential-optimization.rst | 135 ++++++++++++++++++ .../preferential-optimization/generator.py | 10 +- 6 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 docs/tutorials/images/preferential-optimization/anim.gif create mode 100644 docs/tutorials/images/preferential-optimization/system-architecture.png create mode 100644 docs/tutorials/preferential-optimization.rst diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 6d31ae55..932c03a1 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -1,5 +1,7 @@ -Tutorial: Human-in-the-loop Optimization -======================================== +.. _tutorial-hitl: + +Tutorial: Human-in-the-loop Optimization using Objective Form Widgets +===================================================================== .. image:: ./images/hitl1.png diff --git a/docs/tutorials/images/preferential-optimization/anim.gif b/docs/tutorials/images/preferential-optimization/anim.gif new file mode 100644 index 0000000000000000000000000000000000000000..ba15e2b5c5f414c143ff838d7613b8c5e1446002 GIT binary patch literal 271080 zcmV(#K;*wiNk%v~VSob80rvm^0001X0ZRx52yq4ge+B@I1^@{M24V_BWD5Wd4GlvM zClL=6sttA zIUFb&9UvneAu}BpJ{|%p9~&zlB^)3gC?FLzAP5X0G9e)$Bq1FuA_=r203jwUEG8;r zCP6YMB2OqfB`G5@DJv5zJ{c=IEiElEEipbVH3KdH5H18QFEOq!04y*nGBPqZGdOiN z0H8MjHaR#sIW|5yEkZdyc{yD=Iy#;^bKgItJwQKDKmHb(m|J-5TL4^KT$EgP!d!X8T%289V1HdITwhedU@e+q zdt72ac1ywH*<4yd~{HN zbVt2(g~)Y->U9`|b~L_rQ}1@L{&ouTccO=QJG6LW=Xku6dT-WxFMfM(gL_%9dyVFM z27G*ci+n-5d;*Srh^Kurw0#uTeYC%Rj;?=|ynqwEfC;vNB*K9|&w-Gif^2ewk%NPS zzk_6kgnqDun!kivp@mqBhKZeqd7y`sw1;Hchf|M;g1m^Ez=*2Oh(~0JO@@hx(}_}4 ziZY6dinNNKk&BM&jK^AyH;#=~sEuNdj*yy;TdR(q!H!MJj&sY7Zk&%}l#`UClbgnr zMv9eN%awG*mT$F}aHN@T;G1NeoSdVaetMl?o}H4WopN)YS zoJq6hOCGmu>fFh*r_Y~2g9;rg5+~84NRujE%CxCdo<^Taol3Q;)vH)*M$O8#tJkk! z!#dq6wyfE+Xw#~tNOrB;w{YXiW!m;84IXpz>fOuNY~7YTP6Wp5kT65T7kMB?h*$8z z#V`#+?n}9{<&S??^3Y(2-zg1~4{Er(@PuH+cTtl@&APQ>)22!z49a;+;4TLCw9pCrq->KKFnppJ~d_fXyxL#kZb3G&FcZgHYUZ zf;~|1N5g*e-Njirr(h!=4+R>ePEPg|K^k)`w&>zr&M{WRj5N;J5jNT7s7^yCsIyOC zvZ=uj57_9#ML5_Lp^iG<`J==n_87v0GT}gRialx+fsH!h@URa)YJ{VpJbb9J&p<`= za|#XnsG|lpWOCt;GPg{^M>z3J5|1HDxEYT+%JdnJ65;rxhB`%XVb6|yU=ztsS7zhcqw1>n?5NQR*yaD2qcQ?1#6to;9&`d1 z{e+W`JlH{rPd@fwlMf&KsMC)j{={?3KJoCwhd;OQGs!)9>H~$U`>3JMhKB4jp)&V` z!w)w62x*8t%Ea?)oBMFFkCpyJDUUw(oWhU1W-5aZp!_J)4|q!OL&KlGsv|8v!zz=& zq5zzN4``-VY%yG!(NKgT9CvKzMuElS4?p+pQ*U?+;iI2yyvBpg3FHE>52y>uGf6+U z*sKo(;p~(1B>E(P&mSLw`*w_QWE%DeR zj~e)#pwA@y05AzY?P{I9{&0cr?OrOwZag63Bk$6u_;XL__t+!6Kc{o?xzAnm(2o))jLyTK%1|e5^7z~W z0JdM>L%HDOmv0rSP{?!X`k;cWfjq`10KfR7-Ze?B2Nl5wYEY;lP~tpLg8z=B1|ID1 z1b2dh0coHHB`Dz({i8vh_@D$n6eL7k=-fR{!36{wf(uXpp#Tk71pG|m6q6{NAqqGN zF6>WB_X{Bn_O}^(+(HQ*L}3BdBM>fd0RqPxpZWiWNJOP%EJ6@Sq7p~55d>nxJdH90 z{uIK*gy7FK13?dLCL|EtaB)L{F->jgGsY<%NCV1H1^@t5#x8DXL;FFGd+3A32!RHR zwc1rU-gBPg5XLlpDxx32TIU_8uXwDO=vyQI8Y5J z^q~-qs6;19(TZC1q8QDnMmO3}1}X8QAPuQVlX!vzjP#@^O{q#(%F>p)^rbM3sZ3`| z)0*1!rZ~;1PIt=Fp8E8sKn*HL6M)7A81<+~O{!9ns>G5C^{G&es#K>+)v8+cs#wjc zR=28BqAFtrU=6ES$4XYQnq~zjZ~z1vz}2?8^{sG?t6b+w*Sgx3sJAGqUiTW-8LU99 zb`7jx2TRz(8uqY=<*8R^z}Ut*_OXzS>TiXB5ptiV`0cj~aQj>nvw;=^CaHE>em6~+8Q!VaL zlZf2k`c}EFjjnX3OI?{TPVQ<1HP?knQNaLrn3mQfK)oUpbS1F zfDdaLE&!l_4GW()xD`@Db3c4u{sy;EP*`ts6JQ8s+)BPQpp!$5%ijy%;>B;B?{gDC z;R-`wy#NlfkcWI#ybf5l1@?z6OmJK5D%b&)O@PUw27p2t3cUAdmduz30Qsa>8O8&0 zNF(PDPkA}G%Fu@)#H_^c`2+tOZ2oON{4og?|5ypxacW8vP$WLwcg|~mD*(KOjpK?e zNBZDFoQcfnMmyT3*TrrGCT)xCC}Gmueey>EfFsVic}@U00Y9#+kKEliya`akE!1$_ z{dSqmaBlBMH2?sSo#NG}zONb_{Nw!|7a#rzG;>dYLNB|Txox&bui51n7VoKBtzM!6fLZqd@3g43w0bT2(dbUMxg8 zh!8(a<~1mctfZxG6h7pkg!^(q9eX?PDelp#BFtBEPsB&v?0jCJ`D17a02@x^!D4kl zVZKoC9J@*?v}MB>8_NHfap-=EL(Ch_PJ~Jixk>m#AC*75)26kL zQQtxXrQb*C`N8KL^4cFvDGc_9`H_-ra@rrr`&{V2exxwqEa? z^KpWLQR{2wo>u=q{Fh~~iL4(P$};4B3%^1$K0A8*)b~XO=QR1F2mqKo>ha7E(DWjR-J-!eL&mSvt&1f$k+@-=5%Pz2cldxfG#z$Om@QGNh0ZuyWCVCFRs zGk)NA56q=bl4EMew}LDrh=SucoWWXCMB;szIBgSBOEZAO9mkYzm9ESh&-L3m~FSA^yUKW4UH5C>jP z2xR&dh4+z+_McWtWJFnOIu#_+wTUHFuXCes*5@2wn8E7_*^0I&o}!geQ#>Y!Y*tfnJK30ynPWZaWTKT@yCr>_)`xTkm3ektCNqS< zrbc|XUWSK@QztTo8JZ+kW%M{_v*>1UrfegVcR*Kalvj;QIWpQ2jECkmw5geJ1_h;; zBltjKUzQy}2b(mIPD&7**})I^H!b{-UYOYrd6#12_Iv4sd%HMdkJ+8x`BUY1WKREP zl=A3R*l;U6rVf+vT&EBY4%cxlcW`^-Rri1oVvTAd*U*;90CDv`)wrR&Eq)bX*IeMmOT2nj9qwaQ; zBZZSE#hu`3U3A)8X?mx4>VlEUraZb}dHSb-8mL%>S3g;(hI*)ony5eNr-J|5sE+!m zJB3%1HmQ_asg`=F4&bAZx~ZJnsr97+w-A(~I;x~vszLcz4j`wVx~i<&sCZ?nuo|ll z&;dc&sdj_r4x~<$=eAZfqc-A`9$q)U&IoEou-I}iIN@PUE2P0KX1D4&?@z+t@?Vf2)kO}x@mY851vsA+=UjB9uqQ`sZ3bG^HvM&2tpxRpj(m#{XG421cvM-ynIy5#wxgs+qjPVxRCp}ETy-RTe+5dxtQy?gxj5l3sd&GQkfgNqC2{z z+qRo4Qj}Y|s=Kf2RibPFhl!q`9?`*8{^ zjJ{C72W_MQDiXjP{Jt!vM)3PUhA@U``@OI$y^uz{3OpfdpgzXqGVZZqdqi^cp#e(s z#5BM#`mh~0e8psoiXS((_K_XxAd)4#XJbSIOQH|`pbi9gT;ae4=aEh&MJ`|*!Ea;3 zSd6_8FfGT@M{!HTz?N^63&f3iz|%{_A%(v4r8NfkKt=xm1^Y1x;V=p3kq_;WoZ%2| z4p_l;e8cLu4?QQh+%aWw%)TZ}w)1rh;Z}4|utG}EIE11M_@@jUCr0RUe{`J5%?kmp zmk$xT8FH(d2J^=S{D^`)#0gx;`m1K=fe&p$JGGV%laLQ^Cdu;w0Yj5S7-!4>dmaXQ z0{h^ua4RY}Kn|C zXwXB|Vrle530=$!t-t%=(A4Z_e2@$OT*JKtvrjgwuzmSi!GgA)3In= zHy;bSOzqT`9nnw>DEe?FS~JZtU7)+o+sFUwI9kmI^-0GGP$<9Lw)I<5o0Cqm4FQsT z3;J*h@M8$3Ow*Yi*xr51@}dlC5If#1*p{c!Wi8Z=nA|^h#IHjse2`|54c%p%CFe1L zpwS*h5Dq!v!d$@169NS%Qp@>myRluLB0|TpeWE1p+Jnu;G_2xT65Cahg^8lKYGBeI zzPu2i1oAL&_$}B)z;O5Q!iY`aEok6_W;6USu=N>vI9t5~Jw|hMM%hVJ`8y%^RpZxt zzHjSgwVd7@B1RfKNAtBryFKMre!MfbxNh52=q=2u5I zn%?O_ouiU|Uf!W9@^BBo{j5s;QJ%i)#_igy{^}C~>Bkr9U%FqHPP?$4>l5PYy58x1 zE{?QbUka`2z5eQ)UhIng>jsQQ2rSvizUs&R?0T;3!p!W0oa$;W?V29#+Fs_=KGfDu z%+B8Jj?V4m{^8$V;NdR2*xtV9p6tB-?%poz#uw~#+V0&g?^bT^`o7ocKIH2zzQdmF z{ch*{Uho_J?`l0t&2fy)kT=BDh@h*j5`0nu> zFYzXS&mdp#B46vShF(Ei@+trC-5o#k8@%$~D)M!SIEiyOkQ}x$Z}a|+@c;HB!8DG3b@6|OQ_1$~)UPt6x2x^Acl*<+LRFBP5|MhlE?>R5TWq4?CMu}X8 z>f_G8=RpD_z_oHe_jG@>Hy`gFQ1^Pj_j`ZyWRLV_ziA=^p#N8TTp#moPqtj328`eM zj{o?OANi6$`IKMzmVfz}pZS{q_#O1WT)yHy6#Ak+`lMg_rhodVpZco5`mEpju21?q zFarK-^<78gBJ(6aE_7?(^@)E1B5*>Z5d6YF{KQ}U#((_CpZv z2_%35DDb|Y&jp9j2Z#UA{oeom;2-|tKmO!j{^o!F=%4=Tzy9C<0wOT`e}DB!pN(gC z`0_RI?rs_=00WaS3FZJ0#sm%|SkT}>gb5WcWZ2N*Lx==%kU)Xr#6^r5HE!hC(c?#u z7$J-#S(0P|4H`ATIkX0i7ML+*&ZJq>=1rVAb?)TZ(`QT~2RMWzTGZ%K2nkFsW!lu~ zQ>am;PNiCPDN1Ec0m$=r&)z+-_vFz~YEtQmvQRamfHBEjM7VL~&ZS#7*$o^cEFR_C zm#3B-@1ZQAnZEU{3h#e5y(q`_yz(8UOT>Q3b=qNP{2AO!QP#g;Wz% z4;_v4RaoVea>W#v!jw~7a}Cf_R$CoZOdwMgcGwD6?Ug-OVO17RC}(AnN<6L9by{k5 z8srmRr?D-HtizvGbC^L;Tr|p=vK)5ecayzw+EvDVdGK_f z{qf$H{yntdZ3Vn|>~TB3@Z@c0{(J14M;`jTJ^R5AfB6g8{ybAXlMF9_#2et(3W&G^;x2&%Tp<4g z&E-I=Krm<5n;->`11QpQvBK{xOm~6XeM(c}cEia*&(kB$GU;qOi znaiYDGn4sDXv#;LOrhrAAbCwvPLh&OL>-|F$jza7Q=8xnC&&1vx>zFdoaiK3Hs`fX zc;-u<|DlNdJ)$JHDUgwC=`v#GNkrOsY{J!Q}L-#Yf5!tLp7;YuR5`;vTCbb6{lB?n9nWl zQ>@dXDo)LMR)LmurUhkdRot3UxV90fbA4A`lX_RYN>r~vTx(wmhEf0Y1=f#pCG1#Z zx=wcP$2r@QUt+tJR!}w8vA}~Y|0Mg!!B&>8WSuEuJv%JWGD@_gjZLq}>}q$ptYxfsziX)IerdeqHIaD%i{8|#S5w@Tt$W{lruednuk_tpLZ5nq zsOopO>=mzn|68U2H)_BHc8ztjc-l~=SHV}}uW%jwU?N2rOcSQ?NXB^;?B;jF>%A;y ziwa^5iC9PTEipMQyhaRHQ^PBsQHQ1bVHgKV#vh__jm>gnKIZ>;qfn(KkG14u`2sn} z85(k{NL=C*-$%*0b#kW|ygMmR`9f7*P?3#%5+pZr%gOEX;J!S&FpF7qQ+()^DV$~3 zq?pMaZ8O{6jJ!C@nLK0;m7NKU=ac9e!A5FI+D>}iA?(v_w z7imd1kJ8UOGgIvwS^VzVrJObpl-UI8P=jmK<16)(()`Ia@0it@^ECWm9c%6o8d;Vm zFQz+_YZdR>&5sW0uYs){VXw!`r8agsxvcErxVll!es=g~eZN|F_0*~FOPW4T|9GO*W6z>cWlnpmu#NfsNg#UdaQ-+7nRr8=(5~-(qp%DFg88Z zPwz|Av3zS-I=tRi?~TZ3taV9q9b@4Z7}$jwaC5Jk?8d{o9MLXlwIhY-eVw{Kkv?~8 z)qRk6mvi2Y;`KYbeVX(>H{f$)b|MpA%!a=k+uJ(s+rs_hj@O>xO)2>)Q=TY@KeXb> z$#}VQ-Zqf;70Er<*^{H(#d~LK-{bms)fdO}xCH-wkVOw7+H=+Rf~|V)J1_Cpy1sU> zkNu#(tw765$@D1J{Md~zz}+Xa_b~pwY%#+#oU}AFsWq&pHq1P+ zgG0`fLuq5d@7h9<$ir~j!}#mN8jQm#JS;k_LmtdHGcrVILd5HmzeW7RM(nLeghW}p z!@Y7s>XAV{l)@?0y&HtHPV~fJW4{*?#oDq!|9itsbUZ+WH$i+ZLew}|tQk^F#HV5l zF=WJ41hiFTMQC%yJL<*al0_kGIbrleTinFvI!0vlHc(_NO8g~Dyf8&P#nq$6?y|;f zbT?&0BxVeuU%bR`1V`H_#!n;1a@@s9OvmbS#wct#4R{e}=tqD2M}Q1SfgDJJEJ%Yq zNQ6vCgydI7k0YvPaTbM{f+q69|GX_(+fpNs$~$k}OG+JV}&H zNtIklmTbw9L=c%eKh07J8z2HAU;v!VNuAtDp6p4V{7Ik;N}(J|qAW_I)X4y_$W_8f zrol%r)VUKtfd;^V2)F^Oyh^OhO0C>VuIx&${7SG4OR*eFvcyUpK#q!tNn@NzjSv9| zIDn*#OSzm&x~xmPyi2^yOTFAnzU)iCq)W_5xovEvZp5*2vj`G6OT}DF#%xT-d`!rU zOv#)~%B)PwyiCU=0llb*GGxn)U;z$zfdd##(kxBWJWbS0P1RgY)@)7Jd`;MlP1AgV zp%6@_Y!0V%L#T|(5lDf_OacGkJWJplPU0+1<2+8}OitxoPUdV*=X}oLBms)xOo;T% zjNkz5lmPAAPVVeZ@BB{i3{UYKPx35J^E^-VC4(3%#nK<8Q3Zt<4ahKi zagZ==Qjd_*zp2tHt)%}bRS7CxQzzY0JPcAH4Zqzyfg&gZKHviy*enfD12xbHF3?jN z;DIKUQhOM+pxCkD&(>>*ZQ{V$-z=k5=0ek?~6Oe~f z-~nY&1~pJtYBeJ+xCb5xhh^mhHMr7XO+M?yh;WSuC{TxdH6t~M2R;Y~LNz0Gn1pap z14H0~6}eL$Fr)uTP=jbag(h(EupGhe~bN6F6C(8r6Lb z#bLFvV%hFma&aNry$2nPVT0!xild6s(Wl(|ym;^phf_TW* zNyyeFr2%QB)OgTY9*_rnv07~40)&kR9>4~DeOVmrS7f~fs0G`Bg#dL}R6dwkg*^pz z;DRE+h6K=7WiSLv#Rgv8*%Nr%1PBL@rGa=*hPcIstTZ7iR;${m|dv7g`%a>x7}LSr2)64TxC#R zO4V4tRRW*w*%b+g*S%dSr2%=U*LVmAdw|r70Dz@^+mU5oC{O~tW!_55T>6;V_^erM zU{^$q2QE+pDQ(wNzy>HlT5C0gc%Xq($k%Qy0tf zTfm1&NL7f*SY^P|)a3&zU0S7;)b!9*|qr zbyNtzQ$qFPO;rOZXxf35QYyU#yH!*r-~zi%VAaKlG_Fz}m;~o&R5C7tk$_aSeN#d1 zB`nV3&80ewSXDOOtZH%&Chb-y)c_^H-Wcs=CCyST#nLY|*;VGieubDGwwLFq(kQJ` zDBy@pHDx6I(%_wDk#N%|g;Hg9*9>n$l~(AKj_HHO zXq?mNlFnzEHt3n&X@9oqqQmKzHff*U=bkR=c?N2}6Y9l-JEZ>SqkigQR_fYg>eF*- zsn+MH&T1>BYT>hL>Vs*nUgfPGYv%Q8{sU_*dmKt`Yqx%DxQ=VNo@=_UYrDQ{yv}RA z-fO<@>$~HK5fNDGsfQN){br29wL?&XtmBWC7bPVt8M???(JrL zW#0~N;T~>fap|V!ZQ@RDjhw0Z(wn{_k>{W^*=cLH<& zQCTuD7xTWzRL#)xH-Et(*BdNv5e=4ysEr4Czy^DP0&JMlROTEvhTJfRb19`)0C496 z#s&gL^LBbwb>MS#@bgvOQv}|q$(@9fJpqhGS2zy|0LbA+kMo{Y)-rFxZ_aaQA!Ehu zhtudO08oNkNQuXW7=4&zDAkt|2Y(n{NoPwMP*!~4ha4_~ z$G(MCO$wAS+IvU*TSamMEawg9;EeC*I zXAMI**i=mkZ1{6C%7=WVfvt##)X;~@CHIfefOznSaL5PX<%|CW==Ejq@=DSc2q^V? zNR3q8TR7$eKIa1Xl?QIuV0}mc0I&yjICX@F=48hQb=U_bIChRm0D0&Ke%Mnxm-ho7 zYJJ=XzV&u#Hw1my3VVQpdq{bJA7eFO+7sXhzFql_&~k)L0DYJQiU)~^_Xk6G*Q}dBAbp>8RL$>tB{>BpfC7K8daN0JWxxleIEA;@0JJZH zd^q}j(ALf0R7?ec(wF{GDFVMAm3-j+jnL(g{dSpGay?TB=`)1X=0k-pC8ET)@YKBo8uD?8rw%~J6SwRoIC)DY0A*Qw zvTf`3E!?;-A>dQS62K{Z3~NGA(odhfy?h-;Eo}ezFyh3D7c)kzbztL!3mYPaNztMP zj2iRNy%!JPz9RTa`UHIsUp_v9<+5(=`Zdl3*!C&A1$gyLrT+RUnacOD)|XYUX5FgB z&^~qY_?~Z_ZJjL;F~lLl2vq=7*n;lX4rGv-p+! zuDR60J8z@srh9La>o(RdLRvBSFTeo{JTSop8+-g)2s^VDnm4O`o93$F9lyw2Tk-Fhp&IOC1C%y-&!|VwYL>G>R79;80!|h zE<5nS3$NPg>+OsJ3breMZ1K6fy?bK4bN>7A)mz^M*PDO7 z-r0A*KJ(y%FFtVO`;I>S^@snhKKuEn>Av&uOD+HO>?gnhA`O4g=>P%~sK5m>uz?PI zAOs^Q!3k2Zf);FGvA)N@boozL0IU!7_#g~rkg$X%JRu5GsKOPpu!SysAq-KOtX;Rr`I!YIZulCg|tJR=&@sKzz2v5jtgqZof720Y}Dcv91n2H+r!XYitre*7aK z11ZQs60(qnJR~9$smMk8Q5s1o0SSVoK`Ukpi(b=W=SpS>W?pk&?8eCOs)iQ>xOHvb3cijYU59NyUE(E};DyD9XB6(2DZ3 zr#}5DP=9(+aniDQeS=OJIO9(S2xM_NgpwkE{Si>sT zu}XBP85Q70I~xDhwX(IXZdK`2tJ+UzVl`PdRR~T`rqi#IwXc5tt53~3PNU*6scL;I zViT*_rpA?@v$|?sD|J_k<<(?+1uSMWtJ#DKwy1ZsjA56$*wK=DcTN?6b;)wI6-ty>#A)7AoQvYd-8$u4`_^^O#~bKP$EUfV6M(k-X6B`<#S zyA1Qz554UDFM!?qSna}>Qr(pleGzBho$9y2%jGZq{0ra-TUWrVC2)cNbYSU$aKW^7 zFo-n^Vf6n=n8GIZt%b#f;SBd_!`}^Wh-3WM5udNbC%*AwQ|v0%vRFy-W!H=EE8`*e zmB!|?F^-cQT-3^Sz&%EnixXTvy$YGgRtBq)uZLtMa~Y~SZWWK8?6JbZR?73EGJ~(2 zP+h!2&E?g_GKP_5>utHqc6RBPMHOZ-H|xjy3UZp=yx&rk;uXUfhM*1libWfG&ef~) zogiqWQyaW;CH)acDumg3zIU1r~^^+s{h3w~~JKN+}I0OJ7=O znO+^BJ$+?RyTTHd5cQ`+?TSmdq7-9LHFQ~hrRl=D*(a5Cr)XVk-rbsaxlS~jOFisM zT;l)Nh_-U632g~(ciRicK5KuYyKGlGyWJv%wz*b3ZC+J7de{DRxVKGiZxeaZrErD5 zFTn&Vs+xI?@tJF!#2&R(d9rTjhQ~9qLgBc7FYB z^{acK?qdkK!MVO}uYdj3VXv3U$xdyv{~7Htzp|Xi?e@2idc1O1Jk+tib95is=Scqr zdfz4e_e70d@R%dKreDrJA$H~wH@pI-8J=ed?GuYrVTUh_;=t%^PGTF~R# z@S{h3=}*si)z?<;9;Z9V|3LZ5AN=xsuf3f&kEqV;-gvyXw(l$M=0hKU@lv;~@rBQA zyUTLlul?-@U;NzGIPn4N@3xQYrPung-|Y=o z^;rti#ZUHG*4mL?{n?+lR|-lqaT)+PYR+$QlL3jWCyn4-o0T_QrA&R z$V^UKO`h3JJ|#nOUn{=gD{@v#%2gvQ>7%%69r_C}t^Q4QX~1 zDW4r_f{y7j=mH{~X+P|PKh%R4kR+S7X_Y?OmC7l0vZ$W9Xb|G*cj_rZ5knskgFnEM zJYYi~l!AHc=peRbPn{^}q^P2{6rEy+or)QxdZDB)6frPCJ@i99$b%UCf-iKcr-Evz zy6IL*s2i4QWWMJ+cIm!+DSd{iX13}wOal{4!z747AJ{6c_MvDx7q3Pr+)OC2Cf2E5 zN2)&Ass^UAE|eVgK_3Xit=8sL!s4}B>5pRRwo3mMqgI-unqatc=(uv$FT8>$-RiVv z+>}1(8oDc-o@TtttFZ1RvD(^vUQ&G~tE7gTuT5-@I%GwO>gSN^!9Lr(&RV@D;Ju1z z#G0C~nvMrY4+A5Z2tRPr zVy*<^CoA&d*9x%89`EdGFkB%p;Kl#&2kUM&@@^ik?zEcljP5WGi{=VrPX-GyNxiU_ z7;(hOjRPwr)y*CC60bs8uNb256f;f>J1!Q(Mfhge_*yRi=A08h6ckJ4M4s{Dq%q#E z@msWUfxWRv-Z0@xX;7(eVU{BvD~=wQEg!puAHx?QZ{zd=ZvgX@>jLnh!C4}wPZeA7 z{BrQ^YH>P(FzVKD`Npws@+Vl4@$l`}1v_RZdooBh^6e$6l z+c78p@b)6FEMGAZbDk~JBrf0E5(_dVv+qx_a>k|ME3aoPA2TgejwquIDMPO*liJ0d zuQUTyFmJ9Jy05Qx(YFIPU#+`!=-U9PiT0yHv%GI4hE z2_i8@ z{&P;BmPxm0Py6(d9CV23f~l1-OmlQYBOd_TbhiF#Q_r$MyB|Sks6c!PTBo&IuQgFP zGm%*}Ru{E+k+JJ3b+3LkJBPJX?=)0LHK0@pUn;(L^uyw>0#m=$MjZ{TU`_L ze`d2RQ}j+)bZb2ZVQ2rgW(PJ^$5>-~s$An&em-_TN47CHHD&9BWk=Ryu(fNywp&}- zXQQWQqcFG8v|ZnIzo;YGdU9&##A??SMiZG#N2x0iw{8F5Lx(h8kF;Jh+*p&KU;iQ( zAGU1Q@nVlxFl*oqqGNO8v|j&qn3T1V8nks^x9Md!c}-#l{$FmtX+#_BWJmWLlJ}jO z_dF`GjB%eUWHn+Z_8^{HX~Xhq=d^h1$#j37y@6?d2Q}~kTqXZ^f73!bL-b9!6Wl-B;yDoZ)ySrO) zD+P+X6o*1-k)kaUq(BSBy|}x(li-Bl?q1y8-JR_8m37{;);=<`KjzDSCNr5N&vo6; z@4g5O<#^uS;=vL0?JD6b{>bqp7>&{pjRO^mA!&%A5sGDQh~*ZFd*{m04j;#_6~w*$ zS_%&C%tzCcgqqK$V{aDwK`Vn2jTxvnhn6kr1!m(8_@bN9rdoDFt6CJ0ai_9Og@!{HoBR zvGBWa(XYlLZ{gyg#^Na9lBC8GYq$H7%U@Y=Xz_5U+HkR4} z`;__nK%zt%lA0Q_L>h~l8Y@Ja>Y+_dZ6eKaB8+J6e%x+&#tmQ6{lw=mYRZPeP?2_& z=5`#>jv}{u{S~xqIJA7Yu6NB{5~AG-&E4vvJ)fF;j6{1antNrQkk<@Z)azT}d_>bb zH9p^qsrcn5@fNHOb2!j`oy8kwj;LqA8JTS!Srr}KZ5}-p9lLEFgNlx$h@l5aqSE3_ zOp3PnH)p+q6Sw!xN01bqLs~5N82sgp<7_Ez+ zT3$(^B!-8`(th4j?;VA7YZpTYq=SQ%RRYh4<&pDsb_eqx)tn?IA@w?=H znLecbNs4;<7>HbqO45x=f*Oo0DYaFP_%r%Fyo&^u3Vg_%c({wYcX~UcMmV{O^n*eG zo&BpYAFn-s?r{G@keu76bhS1FL>gcCX%;Oe?j?{r!u?!VplSxWIDn8Xop1~{OR?VW zTY7#ZPOFQ@$?rwTDItvv(&lgfVy+EIcMQBQON|_v5Mq8dTxobz zZE{!$N0ptg(ALlD- zJIz2GR2NhQfBH_Lb$T1PwGBd0Zf=esDj#YP&P#SNPTC|J5VIRsjxwejpVQs8mhhuN zOb@Xb`*AJFx1XT5P|(-7ARO0Hj35kOwJM7S57fpU>e#K$UWiNdW>lCyPwEz7Q+nL{ z0{g9)zG%fzO3T>g$`A!C+A8g$(yL)nf)Tz5y(s?T!VLiaYspvz{|896uK?VQehGWGqaqr;8aRtZ;i1ODhCoaBKaGF*f%7(~==)*y_=CwOpZ zVX>^o3A~oEpH%$j%RBfXH(U_kGFJCK82d}2b>Qhva^o}EUbbiHT*(dY+XxhQtF^8b zg6#gVeb@)kF9u^L7BFN7D8aqxHP-3}eqDdX<;}hjZASxvZWcH#sfeyZQ^@PZ?M-*gpz7k7@nzwKy8WoZV8kMCMm7NZ~D|WR8T5f zAuIzG{y89(L~%VOW&3n8^94tB`h$r1*lCY#8`bTSx9Kx$k8H)!)tC&H;q^uYO3dwS zf|2+%wE5fWqSUuTHJ7Xq@#^)8`n(3B;1VZMZ7iVXdx#6?~inzTSlwu8$b*~kk8rL8F zC$)8KXMJuAw>K`~Iw}!!2YM#%?~*2CWL?2jM00s{pni54KtY;f1y_|xWgXja|yic|;9j`Sw8ZXS7*7hygf z(f)fyy$^Ajgaxdy2vK+8LsBbWLALL{ij|&L+6rNj(1?|+t5PbRFcKV5!S=hiP{5GB zV{MV*$Ab?k3))5EPGNH@=$%t^y;>^LCKSI}v>O&aR(KHsP3j7@Z-Zs^=`=$&r}93} zFX+_64w0q4?8pOygp4z&Nk66!4eu8|wF%UfR}LK-q3M6_q9(3#!mzoRO8wj?PFx-1 zd1U%Y=<|RfaZSoJ$0w5jtsy(&+8m-|%Y6X3hHXM^$;V@WZW{fdA94vO;n>DXe=#Vv zx2CG@*v@rPf1s7PqVf9JK16@1A$Hn-;2G$UvS_f&#$TCITOgS9XmDsq(&X@LH33)2 z^_%QtROiTv%YgpM*|Rq5{T5}<*C-oIF3;@7C=!rn;1rz1gA@WI{RHkyFpB>+h6#>$G6cBgDqc5Dy#%z30%*huz87kk0FzgI$)>C^@3fMOTxO$DeS~k)8UL+wmAgMXzq_405tDdzp^ZS&BIWic zWrnY}_JUhz^KLSY_`296sG^jWXEa~ux>P6;j%&^zIkC|wSNF@I9y=ln^{ZF^9mW~_ z2aE$nqx}B^ku<>{{{iDD)fWF3j5ArJk@s&Hrv|Y6FBm73O8H+f z&X*zZv3i{Nx?BD_xO}2pJ&Bo7wD%=CwMz0nW3VsI;AHVD>8^Hn6#4RGNaglW za$V_szubmtkmK7hYS4|Lju$>Nj`pXsGdY`lv!O`a_d96eb!pX^BTct+`et7!fKT>6 zR4MF%osvuS@S_8wp{Z?Wh!3tI~S^i^T(A(okC}S(ps0mQ7T>!6yS`e z3!)7yjzyzSv^14M3)q?q;Q$Njhmv)$;3Bf)*P2EkPruI#6Ln@bQxJS1V3d(~(PC7P zxhZB;Rs>9KNoa(3_bC*?A+^VT248*EWj$cfl^y-YTKGl(K$}VZCcK16kuj85Pa!J> zypzsV=gQ>zGG0>jo_DSxj#sZh7xyQLT*Gb-=5@(ZMz8>R9A>2agAq3|I--?SQm#|J zT(}-$T7I^8e2FT%*UpC`u^xhhoIL!WtfiQq(9m&P^bYj6bU_E^ty1Mto-XMMrhIm( z&JI4H%6dTUz7+Uk(6sC;e^_+ws7@1p9leRjpXwRs!{hq3Kz2LWbftB<%R*+^N#hat z@TBRim;Jl?{;2hLz3Y{-)7Gb(!_zi66ppiYL=u~`4pfw88#}ZL-Ez-+0m{`b{J2Aw zp0ndS&fd?CPnmtee!~oi^q*`l23U%K7lRyHA03ByjGSzw6y=W=6J`eAvm}}re*G~l z`5ayh8zXN`_p+pbH&re@H?6(*qe0kykgGwfxn)!sCd3v`gQtzyB5U#a<^qh zqq8)QP{)b|ICJQuf*QA^Kdt=nX$_Ox^TNd{+M-`(fs)E4$ipF!dMG2F3jaadGmSjp z)1XCC$?_nC_#ds&BH_i;dI)@!@)g|lr^U=cRe0ZtpL!O+qLPFDslSuh%bu21()U%5 zV*mq7U!XO9x}=~WF3H1LkR)aG0{lkj;6Yd{WW%w3iL&odg0 z;auW+R9RN)n9v%$RQ#&`9^TaRaM~++(#5y(e2wQ3>e`#+vy<|I3+Its@tc&BS_+~M z=TVGPo79KZ^u-1FQ9deLG)O}I7qlN@q5~KhD10p>ImZ{lyqaWeo<5=?&M9JN%_Y4%e7N~BUZR^tAIytZ^%fAIk`*4 z{7u9F!X2!mxKAe;JAC~_yD897x0lh3yY$g__xp<8D{r&mBbcQ?RvH2DS~(aWcOS^bF%d|O~-Q1V^!;DacaR$)?( zmFlsJRywhKk%*_6x~DK{HZoC!%xi#QxZ9LUr*=t;2|%kMS-W^au2gH<^dsni7-&uu zs=v*q-~2HBN&F))pUzrW-BqVHR32pg+Ejn}fN+r?vE1Q!!(bmk*1R(vWU^hR`}8nN zjI2}nmCnZGlgsDsP`RqGhz&!krl~~v#G<%@qc61Xx`HHPap76>W2Lcq9eSQ6(;=jTgGETHz78j2I@7R`FK)>1^^m!ymVg}BRvpo6~*-lp-#nuA93FlLAV$-Cjk?}Dyqz9a{|311KA>UPbo-mYC(=RR_=sU~zd^s?+fZf(rdk1O5T$b(ApGR6Rw%uQr z6nWRKKr3@0h1 zw*vR~p+R(78Prs=S-=S%|7jC>mQLzb35g>b%@*QWWF(%I;g3v8Dnua}r5a2jXoN1aIfD-VrvLrOtk1W;QXF*<0 zX(Y18JlUD3t?zF4r!(9gXpB~d`fghvmpyF1S4#P*<#hUI&^BwmH+pg8I2qrz_4SUC zR*yfy!{?sJ{qXM zlzTk{Re;)t8uxp%tkaCqycZm8d+39163h8u(Nxlfk3vVc?@&7>F*|~06vT6oCPs2aQ#l58DO(JRAFFjhqke6~G>& z-T~cx(zV{?ecTyqlP~SWF0MI)mV&7S6xVzAmTb}=3-SFsTn+!zS=Sv4cU*gP4-E=g ze|~odbfstxsQo+mDiVWAQui)|*Ql=0B(hI)fj=GKMJvfbzu=E+NS+lCGODQN#z}M( z*vX;U$Qs$M_4RIGg}{s=#H%0h_!#y0r zf8d3}krIx-kH|CmG}lkF0Eu{*h*%*NTX&&f6O6ppjNB2V*-MTj=!zto2tQvJy%3DL z`W$uR5_Ojx^&lvEo=ER~8wE!e4KEaps2lyNL1qsec>;;XoR7x7kH#g7!54}l)Qur_ zjUgT26|SX4A26UuiD|!%dFKBu=14&=^o1EMmaQR{V?LJaK9)z3oYjhkN|&CImDUS- z$^BN5bekkjUyJt4GO~0d&PemOlnMVG9IfJfv}rJIa2E};Mx1Pk_6J4MSyV#h6l&#S zTq&W1850(qi3CeI2|>C1xS<)yA?(nB1XV)Z`lq9!?B#-$d zulpn)vSdG@Hf^mOp8qE1H@uF^JiQTcFoiV6aeN_)yT+o+pN$MS+KbNu2Zfp#acWXtPkLPG0!& zAaDOZS@NLJ`7ex9U@u$<%c@PI3th>J{e+9<8jEy=^MDSe1t~n>RM6&+ z9b`C7w{lEl$IPQ75OZb zw`r80>7iH9R`dzu78s)?YM=*87VMWMA1+k2 zP(`X=-l|bWbXbmrQORVD1|6@W4lAP_Dra@CWVfy^fZ_s?^=mkl3%HemZ`i8^AIk~# zYXp=lD-Oy}m1^PNAd8BWJ{~0VVgT{=%VnFY=K!@uY`BVLwIWR@_vkh1LnV?JK=Qv} zPEGki`3V55JXCAAShMs{19AguTZ5Sn%gdwd;6!Q-M3C(s>%=L*1;XGD`t{EGRq$bT z5lR(&LtxIwdi%wU<;Jr41M~=Y2nTy*QYkn|zfSoP=lfwT_E5!-Az)%y1G7H5>tS(v zSfkB&wJCW+EPMS1TYgbkBWf5pycFzVU0-gDrV>`38)hA+*92Z{aCWcq53LQf1}8~Y zaZ!}yVbld!H}(&K5!j2N3(cd4=xvL&4^ZJ2Q~kynKm{bN!Bq*w>(;{Wj<#amK($!e z{aCqW4XO`o96T(9u{$3hTlLtR*G1a$t()cC+KSU4HcFsxVGS>7^}FbhYNaMbm3(9m z$o*qMb6DfaAv(0OaZ$L*kD~fozbSF3&8Dn@R1~*(v0dh&V^*l4SHB~utjQp?1M{eb zN+qAmCWD8v>!_?2o}yfHu$jrAh6C6IC)!-1(@fxz&l}znI+VZo*!3%{t1PS=7Zbc8 z0$~vC*a>T5!6*=N@1peR`OMKY&eoxN)Gc?^P`a3p-rVlo)FI@NFJh1(w$!a|14d2H zcTnjeQtmT1C{}!`qNA*{R_XA8depOo=bL#z6vFd=sbsh|_rAvL{q6xeZ0gi$&X1$a zNU#|Qv#F65EoW2dU19I}P1zA-&=Mk=PwCN0G7P)4R!xyUSeo9`TH0ITkq@JPh0|IZ zjVs&3D`SoZ%78uaDlK1@hNP81)f|=5LoKrG{ohlEo0bMgj)oiCJ9^R^^&Ut1(uXW5 z>lcPcR2uts_3LIZ3$idXa+Zb?L=e4Bdc!$(z1M-Rg*Z#>2t)5nHHTc4K3 zt~?;GILEg!J0HUPVb0LXTtCa zW}(dSIMPUu0x18cu};~xiln7L_&8m3WK7GZ>QhFOb>sBV{PYrL`59Z^-D01#*!XLz zNiEfU|r?_qYXsHhU9O>6Q%Kr*VH_zygTACF|n^q;C8xWc^G?)wgQtM4s z>N3*hHZlQH?wz2VS1g_DADHWPpN~Fj`xH^HIXK_5KYv_Wmwq&|;?c^NJ{K3!mk=>J z+_XR?S_x{InL?kdQXQx{9$vRuET^qtqni0Nv}lJlDc3R*0b2MP4w6S7Ne*Aa%9yo} z7@bO;8H<>!I$8*ZdiIiKSg8z;6*Vs_9gOi0%{VVDWcm|6leUWcr73c`O%OP*QrbSL+JB2RO)l1M{nS?$F=V)~46&X? z5Kk|TC>69@v2I*TE^C;6*z6?R$`*#rI4o-_ttN9+C{d@Yyp*sTju5UC%4asG>TkEY zZdamLbY>LowN#&#t{aT3HwstfiH9}*5AchIp}ymkB%&@ z@v3k4Ivk|jL#C>lGN?*BDe-tY^s+weY+dg*JeoLZ<5D{&EJ_|p z5tARZE5pRw}a> zuSwhJTF3kPbJn$9+qJ>!wITG{fMz9>>Dt)n#@zeHGV2CF)5I#A713~GPjd@{arCot z*&cAY>PqRY%T5rdbyyQseP&bote;VtY___c8bSKP%`EZr|BJE5fv z-zKMcr@+jdw5aavG}KGAIi=rRsN~PL|}oey1YZ!@R=CL=(}mt>&5XMkMqnStzS1c zo=^8yc>&}X?;KFW3H}A+Y$0zbGK(Xo$_j7^YYn$3%4_Y?igQug*6zq}4zyrbJuriO z3jx?=ik*0>62$sH@p2HDAEI7knbw}mQ~qG&k@CX~VF8|C^X5w-k?A=UW=xS@$T6ka zOcani#XBmomDN@=QfEv#DswdjyeS|v3%^j}9palD|Up(=1La;Yk~8UJIDZ?X1L zO_VAX2ID+l#Z?Gj68;<)h5iG^A=qZokfv~oSC(T)`1xTSJ|JX5;qA1xgp$bh)gOxF{84_JgL)pQC+?K3=j!h#) zZtgbYsF7X2#wrWke~s7Das8I4?|lC)*(}l3Jk=U}Z=U{R#?>Oz{pQ{x+n?CYGB-@% z!7@Ki$IYrR&AHJmCM(emplXI!_y7RnMF?A$H{Co~S9TG*+f)w;JlfPw>bTp27oFW1 z_!B+d?HUikk9N(MGx~iMH#d*pBf!WW_MI5-p6t5``gHAkDEi3feg1>q$A`m(lYEu_ zziAlgy6mBFg6VfrD004hT>nAuyYg(x#ZmD`{};Vq`x$pzHjVfHpkeqYy`Q06|F>cI zOYhfLthV_eU{F)?cSv-Z;1L)CasA9t;fZkgIAW^4Rm=HFw+@pPZjENkgi)(D*6#bi zsFmrsnbsZ7R+_~vaWsLC7u`0=_yU^hPgmRhF=*79A?KU@u!do@x#4nm{J$E88tc{3 z|M!N$HsR*-Pyz7$pAAE{TNc*88wN~)y){_F03q`FL6p<|o75cEFi<-0t-~4ymW^P> zM9Ym3mO@lh^B-1lJz+e(ESuqcGnShXf;+{Vk)k(yn^BU;Aq*rMluSus(o7{=u_^-l zTXCx7WF)Z~IBmIA>C2W6GLP_{#0?F`@$1legkV9fm!Y^7)K=!+%PO>BKs)z4IR zR6Q(kcvLek&wgChU@%b@-j<8BOWX-T82j5WoIth`txp>E3jfhCP?QU4es(_&o6oER ziJa8F9sM5-1G8wP&~fr)57C98eueJsj`e5{sRi)7m-5%qc^_>M$3;J5lE=AVuPCKM z1wqos^G>Edj>}=bS(P7yxMTW^CGDdSq*k6qj;nE55-Q$N_OJzBU2clwt4WPjVD+eS z)b!7(&lVu3-uGG~HDGasmg~8%6wBAmAH$rS=56Y2FXr*RZNJZ*ARgZ=yZpd>H{_@V zRjFTgMTxv!QMx_8SoLETznik880FgtRWKA-Pb{RGS%|fW^Mshnapg?Hq>+O%^-S%wuOyg21a;@zV)px~NOd<6-TW zx7YEe@u*`?lvajod-Ik)GSeb&Pav*492gYP0-HevZ0;+ATO*JBwKJYBLlGWLnj z%cButKJ9N_ZJu8wOBr)VpPFB6xqW%Mvrray+EJu6{9g0Xp*IC|jQfrU@`RhbK=F6m zX+xs}K3H}TsLeIpmw~;K7q_+(O$6gn(xD~v%gzX$J8Dd&V;Bj_VPl`$ed?xX zi=KmL{W%ea;Gy}-qK}(Zt;+vxxE}emV5h(VjD!s6q}&(mvQI}T!8h?wz>g%GY zVs{c0$j1N|b*uO(5I?ah{TtUvaa&QrpYTP3ti1j{qRS4u@LJ$EBBBswtqt{*5YwFi zrhApAZTu+nM-#r1WCq!cAJceJwdDGT9?glb&ClNqFMcXP&5h@2wvQ!fmRWqO3vd=^x5{c~A#INUUy zSE{lYf9xr0l8Y_9ip_T9+mm`LFccYknSlo?UWfNx0$)( z0Z>my=Pi~N)1`ZhK9P5Dm4k^`k}c~-6uVPXD1gL#Tn_6AxDt{?_z2$p*3IQY8pYlR=txgUr%*76j>CmPIVK%=kCxA*hWI7&N?(Ffa#n8|w zOtYS8x*n5g-hwD$@!~kKe!P~|*sAGI5Be>5h{Nt}RM^s7X>3Ef=g~xVn(DsYWrCn5 zh=ZV1<9Fc<*wS&ch%aP$`*zoZMMgj&+n4)7uM=VApCM53C=pQbu+R=DlE*ypZZu38b8&OUPve8Qeg=%lMVd;jLh za4p8AXhF3yD2?2>D>S*>A;BVCS-?Q-H<&GNzoSp&>?MVLo=c>15@Go z{-4qp+VP@VQ!fI;Oud)sR~$s*yOU#zptr_C9F{_O!fb(ohN;_&73}U;ZJ_uI?iL$K zc6~h{ax1GE1yzxP%aO~ zfSLefxh`dFCv0I>$5z{e_JKLz>kyM?b6t#v89Z6JM7+g=BQRNeYT*%Z{?j2M;1P6s zeB)%+e8_*jz}l9$lwTqk#X`DCQ%#^-OVZNnT(-Zg8dATpGa?1o%0PW;ufY4Y;e6P}3;i1Mg8Di>YxPH`uYcgCu)IXE8+)Us{ zuF~x6o|<|G1%ZRe6SZmI-+1aG%o<)qo%Hh-!= z9Xjj7!DzeIom@+x@P5iYu8D7jik*=R|C;ip?x2%tgL30roZzeB^7=B+Z74bHSpKh z;H0n7ao@)$xlukU$^|hg)DtfHVclPdE*0{;PYzIm1gNm^NC+}X6%#JfquNJf>bU&= zocvo4^4nnUx8dDyW70rV!N9MOZ(Bd*m;&MphQm|l+iQ9*<%>ABLUzTW>Aq%1<^ck*#P#DH^qp7#;8i|NJTshr^U!|*@&ku z5%qJzZ;TP3J}BIhxCB1_Z$2O?RYo4*V%O8+oRQ+J`(Uk)M?JXUkdz>=pV1q%qujDY z&DoM4&OwpfL^%7mm|F8fN4cPdJpt1kjlsY;4q%L< zmV$Jfow%Nj9oQawQyC|L7Jo2}H-Co1NDE)v0qdE_&jc8h&f;MQT-yEc2T~jpvLq5y z)QfhMUnQ7>YY9YXiR;Fwsv(KzmC4*65aWEXo{KSAOR#Tn*_3dxVf-&)NQ`<((we-x zMhW)Y`^X!~1d?2A6~&}4R;aXAIGw!FCoWMYuITkVN%iB2MHi7u_UMD{DCAZt>)Dub z?NI>xgmkMI16&+e*0h>gpJAvV+8`W~Y$sAtZtA`jYSDPqks{-PaWcJ7%=KQ(AZ_?@ zF`@@g^!8o)lVF5FcGeeHjQX>TH8Kxt6ox=!hCirLB+`+tC0VZk+1SDn$mkJ)_K_$D z9%$$Z8qzrwtzD6gqsfGV+80;Pj?a7NAg^Pab6~P(?JEdg&>LQPYBCmhY`>=)~K<{7l ze&`@>Ewt({y&pY@OD;(l^ACDIIkhD9AN2mR19~PpQCbe7)dWq*Abk!wXpIheMo9qV`EwK0eeox;FwJz%dJuwM^I{Hn|@1~~cv9480C zoGWNM(Dn^zE)}%U2wKns5p{yr$jdi`%fBCh9Hq+lVDx?}XosBS@S*&SyuxLn+@7xD z#;xKmwD9VI_y9xHL#fmlt%c=}bu7)1nS z#3ah)lCUOWvFh7H6@_&XSsF2*F)AXunysmt<1f8m%|lVcD^kM;>k^>uHSA$16#CVc zF@;L@s15e0(jv98`u~I87pd(WM-kvbBp#}{%Y;4hQ?sW&Zu|rnf3IpAnM<1Jmqnl&co0*iG$K6|+*;~q* zn#~uQ=U~!3w5+9LsO8&1%bI&@3VSPtdy{EQ)81oiQdukdP^-y$>zR972zwiva+|S9 z+udVZ>QF;_Q|r`X%?w66>Js=tzw+sk=q0TkHyn(iQi*j$ghSausse@wHt)K%lRb49 zVW0+Ac1$X_64N*0VWP5ebdmx)gME-ohB{6TI?$Rs+0#3@FuUrM+nH=y`E9!1g||xr zyAVX%kkae;p4ucayOo-|g`2w`-MgLAx^J7hl?{4yFuS52;a@Lx42N~zS;JnonF4!! z^t%Z;dg7FOPT70SFk9^~QO%#=KZ>*(F0~kk_c+*etv2=jFz6HY=-W=~b8GJ7!fXi? z?X!f6RuU}{kt}rwdi3{E447l~`-*lM0h=T!2RICx)1SJ-hWi1SjVb9J=^O)1Lj&AT zkVKDx>%rb=%7JG0!7|Jq&84mYj_z6x*p`NF-lxF=%+8{vfqao+qr;&u;R7zffx_mY zJnLa2<<3R+e&*rf5#V47#>fQwh%<25-)3Z%V<@R?WRbFK0CRWY+S#`cx4yOcVtiy-Kft_85Cf9n-HGdpa6+w;nga7=LCP*OMAYiWqqbA4eM* zDIcz=d>Y?40zZ2+654j314k{ICd}FY7?q9R502}ukJCn=GPaE3i-DOUDp(?j*q|+c zIH@|hu_mcJho+9Y*j2%to|CUzCNV_^A?_0rBY%WIQ@9!6H=Y%5J&E`;rk**bC~Ze4 zk4C1`r>urXbv&oR22(U6)0QH`98c56RD)+rlMN~}Ce|~S88dA*6CFo0bXc>2l2*SQ_Xz`&YEQOnmx~c1IH|UAO8uBfCO+> z1b!h3&X`XdnNMh$(L0_@5&L5gf@DxtWT_J6a4wjOH5EjR$uGCQ&sb2Qn(fzLeC4@# zowis)H3vLi#K2m5W?!m~n7y}NVvbnC1};T&&gZwx5^+{Kffgk>=eyJEK~SuU3NfN8 z)tRM{zIfYZYR~xvG4S?CcQ|MU{5*HqvUnUZh<>!hi?vM5u`KarHOzB$Z)CD)dBzzu z()_%9O1|>p%?fG8?Dv+bZ((b2Wvh|HtDR4OEIHQ?O4l?y)-IQ(QZpvhB33n2*B@-w zIWgC-TKcTZ*7pb3HP+X0Ii{qKH}=ps)#)}HF*kmVYzkAYnSxeAKpRJ(4FR=s!4pDZ z>R#E_WpS?AU6tAWqnTzbFumv2X6lw|)t2qjmKfIt0oGQc*jn=Qx^a0uRAuA0?FK@` z#@C~5fb|&QWZq_F+%B^9|+TVIu)w~-_ z*%yj={5tdKyVp*>Vecq51Vp*@b@Y%Ua(CtBXs&f&dn4rFwE~cOvt2 zyq|ger*-p0?AY{5Jn_wdYv^VFN`d@byN zp!I~*yV=_QXhQtNmFI%d=z_`nf+g#Mt?hy%3+46l1rN<7FBIo;duguP?p*E+Ubgao z(=fcxx{`oGK9wB=WfTe+!@zCWH_d-<7>sFdOeL;=E?qGhzt;?C8(>ZvJ$NW41q`<)ujLKyV!Ra9lP#6A4U z?zce%QAq?oT%bkiy=Ccr$Kbuy;C)Wpy&UaBk~%_B#e=2Z!#BV~Jol}nREcO+;mP}K z4cZ4K$?TEOt>a|b2DlH&-uD1wIzT&uM|&jSS){peLR>onz#iR#9N1TZT(41B564hU z^SA-!!jHfquu?)WwTOv#Ej+Je8^nG1s+g$ZgH>b{d)<%F`2q0;7h7~K5w8?DS( z-M0)8_8OhPyNFL{al{YQNK>!0BE)Pi)QsQ$G{slvVz$f3rNa$vw)coH*6NZ`{P-fCgo$bH$ zevCU(7v{UAyYqy1RSufO@2Dqc2dAXAHt&mh2OC=0G(Y5*nu^zh123D?qv*&9)K8ng zz5f*xqI6c0_7l*k&(h?3y1NQ3x8E|&rISoPp*zb8_i5}Y^Ph5wLg9jBcXWixv%}XT zzG>zGLj<;D>gO)l;rYddVGHJ-HcZtdpn5;gQrX{zfkUxZ6k&Q&T^i%~Qq4k&z;i%Dp5hCErX0h^0Zk?L z-*_JsU)TNAQuFTO(~=V*a?*bD3h7!KQSqbG$4GHGB6ZZyu!e!&ux?se?+3qjNSloq z5j>V9uJdQpF#em*U*kSH>zb!I-RN58B{=H=lK7`Tb-=^_U(@|`ur!Ri60oY_7;}CpXj1O&*-Pz&umw+ zaq-XUl#_D}+xSrkWDNvQSPMca-bFrX(?|`*zNOf{RF^Rtu_0sDPE=>I@XeL-7w@Pv z()KEyb)z0+FgciWm^LU!^T`)pvT!;Ddt4Hq!LoOPOswu<8B{wjWCMbLJ(snoAdWk;4Cdx}O<%);8A;8)CmU zbmi8(kCllO`5)Q4O?B@X?gmdco|e$L29Xiw7%Y3Yxf~JP1zwJd-X2|!Nur3=h#cdZ z_H}VKag=p?d_IaC*HGZRp3>H`#ik3O&A1wp?!i!xy?nbK44dv>@E)UbSe9IFx!a2WRQ_vI z5rxeqAt25SvYj2oCA{<*;pJ|J>)rAF9}75<{H?^owg2%;_1PsO^!b=HNT|083oaJqg&5E$sI&_?#rtkw`!;(H z@qN23p@!cppz$l1Z8?1Xof&FI1JB74jR9Zj#mGTNH)t%3Vu<6{8%K*&e;#E$r$NgM zG{&;M!xXNp^xZ`;LWE)oc9KhP2G8CFp$!B{lN)XZkMFHhcf)nxqP+>cP0PW{L5*PK z(sWl{+Q4&_3S#~6#u;a833vBXH?t#Nh~SbT=`yOEVe>ks8EzLD=pD;ZNR$UcEiKjR zT#umi_sD=+2I@;lA8p|qr_Y2rv}pIej4zwf!jxaBP6Y$i1pb6ZrGH_Z-<4I;`4inH z!}ywLUQzzyT+=IlO(Nf9P|J`vAqDuFT})Tb)Ki0*j&W7MQdgP9ZQ@rc-!|7Zq=}@| zA^tY)+nb1z?#0)W@qe-rsIOT9&2R7m=ex4`igf$A15Z)|7|qE>g5@owdDAx-clbC5 zgnzciB(6Z@$*6Xq9qgTNGq*U3_KpVR^t|#Y7tc&EsP@$~UkLnXqb=Szp1{P^P}~ zKk23k1upyzY?sC!>d2kVPD; z(EdKSL0YDuL#FKA^nx~+eg_WRN z=rw3QcVQL+O&zK7m(w>1`aPSrcU>)jc*r@l^SQs!w!ZZMr0V4kyEyo#LinRCXN0@K zKjc{c74|7^;`+L2q+=&Je!b(&k%oZU95e@vm9y5H@+$uCruPp^0Iyk~jf`DI zygtDWfTX3UmKKd)abO9o+SV3Y;jznZ;EM~%2KJo#f)1q%TPLeAuqk{REW6fL-d0YH zQ2izYm;{*$b{Q|)OGdr_)J%7$^HdPxP4YBNL^9dVMO~$!{PcmaNVTWO#$mVU5B zZ*u@LXsUs@F{|y<+)W)>>vv~BdRL(I8y(jX8)x2-9GeZXRj{8`D=T1?$y_f+h{=40 z%VLsZhN*3~(pCITDD#(9x!v2*w6^75h?&SKZHHNm#V zrccCIk!Z_i`it?c$aMhoAko9EnT5XF>oY6PW#xBaJ|FCY#;Jx|h(FRWoJ1#{F=fg^dwGH)fY5GVu~V zpn(8e8u|1~U{wqEkSjtiK~k-or`a=#^|JZd19nm+8uvwKmavZD`%SWK zD2qZ9Kjt}HmhDD)DMXozOTlCHlH$+jAi=#e9J5-p?c+U{?k{C08ka6pC*TIns5>0t zwN08kpP)BBk@G#d?kls-HxoG#Qhnu{G)AjIk~b!iDGtrsl-TUU`mZk&$3#-~-ZimSJW$k= zP?P5M^^jK7yRh?YnNbV%vg4RCK`HNCPZt{&t0r10GG^afFkI>`=}HEH%!5U4p>G9K z-3Lw$0~blaT{62a*tcgvoMPT{g>LW~W8{U#z`l zRMd^%uR9E#GedWGNq090NGdHLN{NK1v@mpccXxMpH_}~#bci(08Sj7H&$FMs*E#F# zH@@)=t_44w>vw&xk5t{;uRFY2b#ZEzrV0|bzWl^~0DcF0A;}$j>ZuRbwzFH&uO32ga8dHQxcQQ4Y1QEC z?i=Kp@rqw_M+&ms=V5emuE3n_33DM|{-2=a(v z6!qXjHLXI{;|;CT3vF--ZAuDlX$oyy4DGlL?Q#ia>qU)ba>{lN{$h&S-4r&q7&dVm zHdVyKt{pb~C9KZ{)xi|ip(=c(D17xcd=oxGHujM#DUek_EL zcj@Y1a{bfpZ;E_ejQoB3PrLsyNjPmMQUW5Z84 z7EM7SETF@xXWVzhaFP$##VnG8eaz9|Z2W>66l;oh&y`U769gyl7hq6Aa?%wr31*5G z6pXEN7wfDad0u2g(T#jT4W_yyD+^$#5k~jbPYSgLX@K2BT|tWuAfqgt#xCSsVRWOc z6u7k(L4p%O66+C_WP1l%qz2(di@;w(6ndxMc&C&trM!aWL{}|AwxWpCmf}dcQxUt# zQMlP`s6igyNw-;8MqP|+!^m6SAmOF7SO*aNO~4`md6*>?zK**HNWb(3Eq0}NqbJ|z z;3T+$-fN{@dxHw8ldickw}lf~gK@u-fU<-WZUAXE;FJ{-cOwT3EH}_M*EB8kg!?;= z>|!j~9Y?@s>}mnk1o7 zBjI9`M;txHTR4F`7+ZLmL%1X-)*)+(%( zDjYa_C#%MV&)ynVf8L2EPOTLd(8^!n=SaR0vJp|y*wtw*%2?+<%#8%k{(S!vay zM&RxO#-z5ordD%zwK#dBET*L@_ zhPf3~_`2qmVdu4b=dEq$;b_MxTe}8A10_l4&(uyNqb?MWuC3P26SDSCn4R;XU10KV z+_KK2)~++rc4D?JytZ!g)$ZleuBVkQV)8D~aW|t!4+>Tf7JDNOdnfZ_50`2ut4BAi zw3?K>S17G_U9|W213q9iTCl8FrmR$qEuypcn! zOFOL}$Qz2D%mA9t+Z?x^3?Xn@72o94JrF|Awk{lEt!IE`H|maD?S0KQ%&9sY zc09!PzW;>>P=ooQ=k;u|{lFdq&5`J{*rVhK){R^?y?xf15UB+&Z*c z*1tYhzezr!EjD3jII-8(_oJ--ux&zib>h{-#JOspmB+YC+USbWS=#@DV`y81QLI#%M~{eTvYohghwiw0)Xlbz0+Ln!3H4 z_Nk8Ec*cx;dPr<~#%P-LsfX#Ljs-^X4NdHun&CGA&w44}W6wBd zxx>DxlzvkR?^H>zQ(c?IU;PGq98Dpg)oP#7X`gyNJ=K#o2YH-TBAYX{>oE7MvoN0D zb)Q#Aowpeulw0eUdYUI6o0mPBcYbRB%u(l(z5pd(s7@R2ZkzN{>vE^4^E6(h!dhH> zKf4?@7a~4KI5rpMI2Y~tEi8Q!dNP&h*%_!-_i=3r`Ef}ybum$GSlCM3Xs08CMzfqxi}WVjm8wh1TXXg8+g4Mf;Pp9N&K;5u z+f0;;5#o#MYTNVV%`~SIL=ijl9zfBGCGm+RA&DIV^$B^-9VD+B;QFlghZ^08*;ght z`kb?d>NV$v&7-1w7AAY|y!I?3&|YLrWRC#_&Vd}@eJ6>1XOsQUUi&T?`)(ck9_#yF zzxSPg11CjyH8>ZeLlJOc)Ce0Fz=_s1v5ajcP9Xf9ejMD6_c}<*I7sO@NLxS1_vmfniPQ0HFipJbV|8W!IXu-S9N+-aSGNr>ry>CT95AXI4v+aVah`4bj6UTKHJSm z5HmfgDnt^?Lb{Qlr9?;G#X%;GDo_C+XjGk3xL$Zf(Zy<u02jY1 zP^2;3s}s*Cn$wxOayN04WFybnU+=`XpMbrw<*BoDlut>sG67Ny?=}(}!65Xh1oSRc zonj<(2aEs!0vxAHjUe-JWnJMQyjM;Jz*PyUU$#(eCcsZ1=`~zq0B`)_T!-LH<33R0 z@O9-1WCDKTFpzRVetBIVIsh-SkdldSK4t=RlL14@i1JxT7XSdZNWB<1Yw`R%-Rv4} zM4+3ZEdme@vk*Sy^pr%E`)L8#RfsZ%3ivquIoXsVeyf;s%n%ug*yIZvNZuhqL#$ z6OXU{Gkd$Qj>Z~V#HW9`D@!Tnm*S@%$D$H}A3u?2Fn6e1rmy>ZcXcqoFD@X*gU~N&6>Uey4gKnL7|k{cpJZU(+$~6gUOs-NuG%3t#vz z;>g@7|4GN7KF}DC)D@2yszXS_RO(CSV)!a-)>^=Tg-X-XL$W5`vZY$K(VX;Y1@ZOJQ%fEQ!rJsuWpVKkDU-m!7MgGrl`S%UTbDm(p)Yn%W&CC>6 z{q~hV;kCb6fV zF~e24lkuol>lHx4Uxu3EV+EvHz`|wCKO?DMn)ts}Gr-uLNd8Og{ySWD-IF|jL{vK|~~<{&>Lg10DB!jbh`c2WTAVNqU!K}mrF zxFo)i+sEb~VmDiUQ3-27wkEB}QAOLeh)sE!L~vj5nYZb09zhcNkK83)OZIJB4ha%4Zj$le|wL4H9-6($?94Q>ml@) zqyZhQ@6)$)uXc^Kx0R#zy|AM3FRQ3ep3hdXAtt|;n8?zdWg|KDe5WSMpQMgm4D{Tar^<-lU6 zp{naN38=DlWrFssQ4rSVh_dAG{=02FSmiA0qCLUHgX73s?Rv~*0juLSN&Bqy&TC4}{4cOp=vQ*EPpUbU*5yDA zDyN@EehRg8mRTp}9*ggnm8^&ReaY>+a$BJ59MY143?^liIrAzNBHN&hyzBWVF1|Sc zOi{0gY+J);eTC*q@3wba9;_`(uY?S_4VTvq({DJU5lgw1=`vRFGi2KqJdZ0T=hX1W zo;AlT+?Q7w=L!>GSIn(g%wsQ!4h?rV&+AI6qaAUKc&AEB>RL1)zvbxXHoGV#Qy9fU z3W-wEF+Rtz9)k5(hKFzONGe%%DaypyK+7gESPo>L%Yb7;W-PGP&{#ya$0PE@ELJp% zRU}nL;>rBpva8*7Gud2HA?I!9Ip|XvpU(KZBBinVx=Qhcutmk}Qn64mjESXrfuu#3>@j|97;^mG8+~M9N#A}4Q+JV3 z$uJV#7ZPqBSAl$Erq@PCeQF-KkRP`<)3GeVF^Kaku*k+*qZN53qw&HZB!`XXox=Ae zu*dI`biFhSh2G)OU}wI}Wf^M<-If`J=r|}Y1A)?4bImx@x&p&hdU+F@lJw6ic~b2M z;vd~ulRIJxq@EV#LeSW9=h2G!31$^Yv@o$cMoPpE4wStNzPYu`LMLX6wLhZI6=i_w z7K*lCEf|b70DmQ_%Y9co2+=6(!AnhKXMN3fxM02gD^6{#P!zSfFP>7Mib?mqN~i0q zS}2m=YTU6Z%kyDk9+l61J9!!iMGG%+1oA$gWLs)mFD(#gTE9>GVeM+5HwSBIa2XRb zeg2bFdt4&E_T1>5dOyiZ`+Y)r7yq$cCDEYDGOu;;W1b%BOT$$3#;lNKJ0fh6mE3}t zp$osPSTaQ?XDA_Y)5n$$iJ~%Ay8;=@YVY1nkG>eV|CxB^A14JMHFojV)!RU(GD*HK zBEEc82^-h?ls;h6sVqyrtyJp8LHyb&<+|yU+6P#|Fs6Z+EcM<{Y#x@;mc9gQw}nAf z=X#Qwy$ci{=AtMzW)?BvmC&=xKotVZ+jgJ3q>6Vzm1*N+!QB^vie?NuKbLt~$?syj zXgDYkwgvJ)lcl)0MHPS9sz^@M7)E?*>kYZtC8W|IAFv9PMB+hz=I9FRF=Xdj~^LZ4BWX=Z__gzoK(%{?mJPygDJwcG$j1k$JFO_bGUHQxH856rwCjA?BNZ}o! z1aC>x*)^uWFX&eh4pDlG>;Kxmn_tGStlnj5_*(YNzlV`NIrORfjcjSkNUq^eX!w+7 z2gls={?kS#4)Aa~_liKnuVuML8;7lk3~+HO&WUYvxlD4`^}8F%fhR*t(eaJctLtV9 z*A|~Ik*fs!jor&W_P{+-lR4)vZg(fuR4qiC#4v?^NX|ug@5;Lyn1oWDncK5lA3N7K zo9i7fBAlY_V~QFds4D6e8&n^h^>%~4bf1QQjI5t%KX@3V{Jr@z^ELYq)klSPQHg`dP}?Tzi`yQu-{(Ou+O~c@F@o&H2Uz>>JiEciFzHq>RfpH?QL5DB$bk$1(?z7tteeL+2c!G)iCvB`_Wu^I9Pa0~!J1 zgoN(rpL&+726PWToHW11yvjJdKvWAB8bdD4`Sm$ zC*a~Y0?)&6`?xK7x(xclyrC9C&}#?-&SKzNqh}$kD`;`bx7IO`$mfd(u0JBgV=<5h zPE9S7JTQAawTVN*UA)KkNaOS%8CpJ)Xppp|kYI#R<{XRKA)=ZepIaP5>lZm&i-=kl zLopGKp%p%S!HrR(KcO;j?<-^Dg0wN$@|zK50_?fy+&hEun~A zy@)-R|G?$b+lX`G$p4a#`LA#po{m9rjY3O~!f5_yItH(Y5NwSLugB19Mia8&5+_Gf zS4WcqAG^-h;?Pvag2|2wJj=;?Xq}j5pnS>%)Jj*$%7`f^O=^80HZ1;Ie%egA|xtb+8+$#C{ zDJe1|`7iJD&xLcJ9p-Dh73iwus@ddb`Q#cHQ0NIODfoIw;$xHcMy1HuE#EYy2-=b#XHyutOcKqS9_yBu#9Hjj zS|XNG?A}rwZd3f8v>>mfxWJ$&!m%jPt)w!gq}rhPT}Yu_N=OB#g#$D^>XD_OZj6;IfZ))Vkt2)Sq-I4wVy~$ux5=TMmpzx z_2zO7$4CsTEipR`vuOUVrctG$IiyCswS=0j7It5SPgczzO8ku^Sp{zEuGD!1qfmxc zv)()8`=D4=p*Vr-McKfLtt7H!d1#J6$}GTt6T3e={DI3(?hUwI2*g>ncOx;h-U#GV zjb0Cp-emtBE`N2el5+>Uu>oU5o8t_d|2teZY<5YlGT@a?MSR^y> zayB7ybUVq7p&;%mPkTuMqBI|1+HAb~z4%MSTf)KMQWNtl?G?9KRb^^N*1pU41K}2~1IdRmWbLuwI+2cG#xJaVfpN7?cvB zeYGhX?CrB1iZ^43q_PP?>F&{=ZksDa;0$Sp0g$EG!NAplb)|kH4=`}76Xp)yFl;{% z9eluSCw-CC`7y0+qcpbXp&fC$V`LPOPio+88#rkd9HTnKJUzU_+~4Tlo|^{AF&Y8+ zj4&3Dl#Y$C9E?!~nd2V1U)X7{lN4v@|_P zIR9Y`e>)!khrsq<8N>fSf$je-W3V4&NWf~oG#N>NnNP5TBh^023t|$CH#1uMzj6A| zTHsQ@ZCjS8f=K%C<2b8|Xh9;)aPPpcKH^#e%U)=EEg(GeF$DrM zaK9ajpC>d-=mV5yc?&%ucb$qpnvxEA_QB_FC+R$HiB4v_jm|&Xu^z8K4r#)*Kx+F@ zHc;px{1+|F9+-Jw=IIF$Bt9U9#|-s5KPZ&;YLOihO-%UpE-f7we^qb;%ooWG3&JbW zJL!^D$}Ig2`&KNvrz$fk=?~Rq5vxYoXOKQ2{>WBg6V7WDjs6q@rn&Mip6B&7NCrPU zVm1WV;$t|62b!3nlnc!;;-`wD>mK6xqwbZ=k!{`&+sT}mhMr0tn&@Iu7);!Kkl90X zuaKIEVF(wr-!+_a&EUGq6@~%RIQ7?~WP(d_6t1>2utB@;+jZ=etEH+5&O^j>4}k>0__3D2pRw<_W9MaM&phS`A%BXKUIFWd^l;^uXs3ZyMYUAUGO}uLIBo(^0a4EK8+c(w0b2 zPQJq?mINdE7|s=2MR;IV2rag=iH`je8F zb(eb)Bv!9=Ce!&Do|5H;)oS`SLp*|XX zMafCcLDCW!W_V)VZ|mU1aYRt%JGADZ;fV1Gg!q;6|`##tJ9Ds_BwKYGsit{_-g zlUV>owM$g!vjSo~f93?MH!oGFk@{u0B9Pt$G!i2%hRc~Wrl?2H>Fg+r6CUiUK48V| zDn92y0!WjQ@@>T$sWA2`*8AK>(I6TRgF~fz=s|K9&IbkRuRrWYs=X7zXV83VD`tAm z+h-jmj*6|U?J3RCu%9jSOJ6fq)rOuqCn}@fR6UHeKsqmQp4(bB6<5bv=94*Vi7S)( z)q%l%SjnNHR|Q;W)f0z_%zmhv4|y5iut%zuf-90|F4uI(dR+BaN+Jh2^lvE%xXx;5 z3*=(8J!$x4J^ELy8lIAf_-^;FlmuL7-A~7?jJ=#FT`Il~(Y0@9xI8N8%tS~#>oP-M z%_>8J>#RLkrM3>eU`6(eK0BJUnSA`eQWAlo_JhISmCS(_jC>3xGY9;6;@JwAtAn>@ zY2dM|7^`aM+bH^vu6(^B`K6Nf`zPacCs%6^B8Nn^orpyPGwaeeAPdw;R~W&4xEB=k4YViQ4U!Z@4!+R-GyL z0v@a8yo2kkAMU;HgsV}L0z4|9s z{WGUSFA++X(z}@cyIK@f!Fswdr9^5g-_V`5AO5#z@{<;%06!-asmJW zvJrLZ0g~%2#vNl!Mhp0X3gH{6!k-)3TBb zwoO$iDW4ey#CcFgLX-UcZmwXps=0zc(UbjbT1WCqtv(LG211RWY$ z-r+$tC#RS=oot$)LMmDfjub~i-`z2qu+wPrfJm<2-Z(jKNaC^|fyqX%P&2<~sz+bK5QL$V!)tX$f=SzX{R4x~FZ* z;FX0K4T{cJ&&-nY){Mzdn6Vv1n0^)vDR|0}5@4I$3F##Y;d}Erw05oU^G)gr+8I%E zHt1j^A!$h7o@~K3?HcM-aQQ^bF)tnZWtFJ29RJx@&z;vC7jCdKcs zok88dZPF#oH!wV+vTs0;0aG-Tk8NCXqv&(kKMQ)PB&_}3=$qqC(OG_fvUtV4+E3U@ zEO;_U>z8}@3KF_QahBA_e0&k{RgbtDje^POYg#IbZ+zo0`43T64`f%d(_ve+i;#~> z-&tFK25+9QMF)$SzkeAIqr3PG>CQ zeNe`YQPNi#p&pbEg;HpCKNbmjqr(WNO~tTG!HF6zuqj(nbT!MH4(Ak-4@1X1c2Na$ z{k_(N$JWuu@A|Y18a5xFafKF}yp+uQYA$T9aPUQS5a+6gNWz)*z-$cp-i`N!_>G6S zTF-$N(Gn`-0S!zO(~{1@ZIW`VE|_fLfRbe8rhVrn#h7&Awa6;kR_efXnMlr0rqD;etFpl$g! z&VdU40d%kF$PLB{AdE^l?)d~bB9)vImfO@LUvU}#?K1Tl)i_MT^EhSSy_g`om(uA& zB71A{>|e2JlZAlU$RGZ90bPmeze-|T)-)!^RmvwmmHZSvGPucp*{Xgm{FUNZSHgS= ze(e-4*|%v`7%DtK)8Df4bm}tczCI{@)mRm=VVV&c})he3U}7 zkN6jHe3)<D5!Koj3YH%IG1s zV(c)unhF~=;Kk$CHxrtMea};LDvvXxliR7lm7lqTSc!a+5&_H8Qk)Lp1*|>?%xDAH z!L37HfRQ9koebItxDWXgex4faTtaPEQm6>qLJjCxueb>VNugUj!8W~~e7HAnz(8j3 zrGU`yK=#{2+!0L?KP?UrngEx@R{?Wo-V5((MUw4CAIe)FYGNp@5R_gI%IE@RPJ*&F zLEpadVZViP5&M!Qcuz|EzA*RYN%9qG@)cY3h1;r9#C|eA1a6_ew`P8_Nq$Sn zBMwx|_My=WXvz+7Mx)HD7eImju_zQhQ{LEh*7EphHGNVLCB*q7`1=ttn<|<9n6Hb-YDH@ zguQxUuMxtA{;jhThtCN8hd9DZ)vJr)>;KSM|8Ee-{}QWCjwWr6CSQt%<3?(d7+T>N zIBsNgjbTpyD^~3q;k_8sl^cpqkLn~9#)bZwqxMaVzz{rlViQeFG6mKQPZO6CMC7_xmD6r6< z`6O8XD^~r7I4)6HC`WzKN-XV3u#BVeAYu7ZocJeJElgmjl@z4U72Nz!;wXGWKHhfP8Vu0@u(3yTg;XU6sF`A6Ue;sGww8}c>cLq0cy^j zU>wTJp_^_g$n$BW;e$#e%7t6F?aQs%=Ff;Xv#p7zGeHdTxP)U#G+X^vcYE@yO+15r zUl;^EkAGEtpY(eD@>3aMo%M`XzRkT~Un22$J`&vM$oV&0H= zG+m_jN3vz=yElKFj>>$VDBkWi_G7-5Y;NL-{`zz?HULKTHM#9BWMeY&MJ1)!!}aP$ z?QmUN)0~EUNjD7y`0ba%-(^f8^!~Cwc{zMuJ|TG@?>boRPvf=*GU2= zI4ydRAUID|g;O{-7DeXHFjlKZ=lcg8lY{hx#<#<7l}x;p(~}DQmEXBqLf^ehA%wpz z|7`z(xG?Rbn_gl1mw=?g4ELDD!pyH3#8&zra4-_GEeJxaY(q`*t#TtdNm+7YNevFN z5>ZkPGE=XY4l*+LNDk9;=k*WM3I~!8Q%h@>Y*H#S_DR^1YeV#p6w&DcQqt(uE=LOO zG5`R6;ql>SB?Us4yqL!^X_@zs0X{pxhB;fI8dIS)y2j|SSFyGuX7EIkZ-_Uz8fNW2 zU3K~RafDvpPc);Zkho{C`7&g*S29*CwZaYE%CxNWx_AZB4HF=Z?^fq&%j;E=d7sxO zt3{sIFKIrOJ0R@jkvqr_WzQXY7PFc&%$gIHGs0LSmNQD-vzk3dI^&T&j=xQwJu%4I zRyD~Be8Qhx!m!JomW8B0o9W}`h?_x{!7l$MuC-P-C#IavHE-vb9y|Z&lm7an2_nDY z(mT|@SuMoFP<&~ED&rf42{3~Jvu)x2fY5+{y z`@Hg>KaI=|rtsz_oeKF6jU4#^Bo>*rp=ncva57ORBoeJW`4|Q@P9zqM?@9@rFSOp1 ze9Re0q1E12CDz)X$dvMh|1|!60xAhTh7pj(Ck^2T&4*)%G)X4A9^_Fe;9zr+`Pbe}(aosG^1*o3@T+UQjX&ErQSuGT3q zPkbrx_h$-xH4e@OCjMdr){kU=BZYr^ri2pn{*4X%pGo11C^4`9_OJLq4O!_SW61r* z1{~xo-U=2zD&Qx)rsxR z*N_l-nO0Rb)}_#hZbDv;wR}1nRb|7|QPjGYT}GSZ>N{y^wBHDq?x7TFHofp$#K>@K z0?}qpP2ny7CL-UWkG=JRpSq^+zFqDGF~XRVCU-cXbJ(1jO_9zwH|I+kiY{r}dsi$M zY(^4aO@9tQ>EykZ)sR9m@IYg<(zO&+8w3*}{c1FjZXxJH#gBgKMFkx_aU#+9 z!Q@czKsZ;4Z=^%m&7jQT_0?SU6+yH%^x&WgNk(t;r za;n)Eh)$Xb1|(p?K=rrpcMBat_^In_hGbGloRc~F=Zd7Ldntdnk^Xk-x}WKd!uli2 z@874c|FDq)W?#ckUGFJv65f?qtE0Nj`XHtTu-fQmMZiy8S;~VBOUk^EZHm!8!Im|E zHQPG2rAo;9O-@NLu3c>@Zm}0F(EyN zBH;ByX%hlopMx~wDBFed2y5Q^;;c3IH7Nj=vq|3`5S;9b_Ew3#3NVyV*;Vs7?NmBe zpiHxn^TqpGbt1e+5YoNSszyjkx=uxJ)|{$F2UyX<`Ni{%<2pHNB8#3`lw?zGX&hBs zjltUsS&d1~-oEQr6L>0R|c)Spv?hn09R%V)P1pn{rt{%kO0g znBsm}gZT=-OWgz1HJbgFV(+D-VD1{Oh)WV0iyP*tQV`@%3PP%ppt?3o6z-<`E4sUM zeyhASnH+1r^XnIrOU8chWz&B((q1w+xzj1v=WJ-6VZ^IHJr+EZv%f$x4iu7>@|~{@ zVGPy_$SdpslrvDhi%pf)(FJcFj8Q*~qOHv;c4BZ^;9U1EHmhEP0C5$TSu8Kkh=(A! zr$t3Be zlaWEQ+6=@9n^@(eWsJtM4dCB3JSw>^u@iqMwo4L3!}BhRW}=K*E^k6o!~z_vh8jwu ztVg)77t8U|j(Q9?SfU4UFd;318c&Wwjt5Uk#6Sb(vslnCMriRcUnRBpvFSE-jFcZe zml~`nPg?06q{T-w7Ur-a$+ZL)Aia!x)w_eiF{2n#higXOojOSgEq;~bq;1`&L`jD$ z>)bplg*rT4OIg(L5h3<<`X$2;C_&@v^~>a`umZk9<#8K>pDB7iQixT||Tdp|Q*eRjkVR5WUj zwLc@X70I~AwwQ5VWjm4-ol&bypJJ)z5RCtLiL$E^?xrJ3{8Xe=yDbnB@X?LQQ(MVC zS*yc3EC1Y)5t9nzTl<`|8k_ugHT&xsiSJi&qQ-MmfCKH=9GfCy^pZp`8>M{pv0^!M zR*LkHS#9aLXWSq2HS^Kxb7^-=I-+-mtw`IcS$9J%*YKDGf%C-5pCaryh9CzPsY;*e zD=^$_2~h>4W0U!ESy=dF9Mer zb2BimoC+lb^z-XFfq0jK?`yM033Q#mel)^V|DwHcg1N1(IkS0)-(-PYVivtPxe}82 zVy@iuGjIL+d|sm0Nx6;(!O!)(aY3ycc~#Hi#0|w?H_smA557#d&J{p!>h{;PyKfv?a0G_Gf+pnIjA8&NB&0gob4bbc-zts;5w}K_O6-+YiJonEE}qD75flFxQcl z?wat-ux*6a;$)HlTNAV99ULaKGTF}tQSHNTiQM&JFRVM3QV{tqYxh*R{g=@d7W(jQimr-ljrgarI(*l{jdp4b3+zw zWg4`@>~wWkf|o`z0QGGYP-&Q1!n0x}Q>`BEuU0t4ifLiq)%U zl21dLflg5C-%fsk?W1oB?-%Y2uN>c{^V??*&yx_`^v#Mher$AD7&N>N&s-a~esQ&o z{CM-~EM3j@)$AIm_BN3|qc3pqV3TC^v!lTJvqYl#T^6Ie5AQObWxgKXfe^Sg#Yr&a zzH;2{j&qYuU+1WlR6a5zztz1--}QfWd)Ue6F4LdE(fLgI43*%q79UnhMk0}F8|f5O zOTNeXwPT6ws9NZQNniT@-o# zmPTH&gI|@-ZnPhIYhV$(ozpi=bg)6DryBy~B#)np7E6I@sEY*HR-X5k-UvTlBK@$} zd}ETCNubqZ!CgsQi+LySey>NrVYaw#v;Tk9XQP)rCwzy&G? zMNql$=83ibUdIcvw9}*m!cQ+uiF`lzKtYLW5E>xyxu1rTUvz`_mVmG1kRLh?5T2Q& z?E)J8D>G^M1Hr&5z;ehxQ^8+O>&0^y0*-p%RiK|l5}%$6m?(<`eF_1o8p!5=fHQ>v zWe$XG0*I=B6jDJHQivYRfl$~FfRhUXQxz~Dmjl`q7%>zGqYg^53JNL;cJA>d;`Wx6 zws-!9Xrn z!cmA>#45@|0d~xU)V@GOZtRy_*sv-9B^QEM7LHdJmH-z5y9OAXgA%AmE1wmtZwj8) zAeq)cCOr>C>{j3dhr*^1C#NylG?0<14OoAK9W}@?D1qrU!1__x=I0=OE`+6X5G6H= zz$OA^6#&ggY|I;#R0DCV3s|~^u;eXqa0_DR!X^*~w{sx?9f9yfBZ(6lf0YmXWY!d- z)I%+f#|0MP0s~=yNQ`J&QytwrIjQX1n3*^&>5FKXX&l(>l?+qdw2t z$&5pHE|SH{ncRm|dl>wx*~1yl#kb(|PNI(mlcyI^LcpNMw*+sSfw(F95Ph1&DBpyp z!i1K=gtff{CzQlVxkQH_iR{Qa-YiOQUHvSDwXS4CB)gM>i<3^{H8~nYV(;D)@Eb)b zzfNvW_S#BHyGz~!rsNEp=jx}t-AXPPPKIw;B{!#(6(>~;nN+JNg{LsrdcV@qk>EX;cNsVoS9MlendJ$^h%3L$;E6caNGFM6W` ze9=KcakbbPMJt0D0bw;CX>n6-8ka*BA+t`|c--_V@kA1!mf7Geq(%*Sq3P`0 z?$7$g^eEYxE7=GJvbgLb5XM6=_wsv|mYke9)>BPR-un+Ug8?1=;Mhyzs!y!qBQF!2 zNMOxQ)Y7El$dWRPzg{FUFD59X$2 zT_VG!S>$s0hETeo6_=krg(-L>A97KIo)3}zZqU2P;jF@xOezL4D<+P%uw%i^P%+_& zE|dqxWf>HrW;OvwR_}wDlR23SJ9zzPpA|x8pPuWODQYr;45|mq| z_|CX-Aaw$ZIqTnk}QHDZLB)$huIBG>RTZ>F2ZU@DlR9iYGyicOr@pB7u>?YOoUTrxba_hXiCpS=5z8^yjjep_b5Q5qM1)+g2LS-HO0a znyA&vD;PNQLu)?-OnFp|E>nYKR--Iku2KMDe&8Dsslk}3;c&-iElnqbA=FZf<~#G$ zDrnZGeuB_FRJ+$!YbRCb&}5R(WGb;i&h#r%s!5)wmZ&{s2+7prI0BP>>YqF28Q3Q5 zxIrAGBo*(G-aEz0Jg@yt9PyskpPaO&5Ymuc#mG>FFJWtlpWPq_MMiRrQkl*z(!}H< zuEO(ae9eP`G1d5ijESYITv)U5`zMH}Mw4M`lf4!~ZU}zFgAq|RN?I$5EDuU-Y3=8p zCZXu&iFk?EQ=U6s zyCHM8JARWpdrfM!ZA_a&b=#U|n}B1RAFy4XrhQeqo!_k87uqfd%WhxkZs*%-hXOk; zl$#x7I(W@Ge4rgNG#$&`9Xwkd-oQ>tn$Aw?PHwZ#uh343?9PtvPOhy^FJPBAO;@{g z*E6#&PiU7|c2`Ar7w1-&2e4a|rn_9ao5QTz9oj9D-Cfq*&A!#`2JCq#?y#5XVKeJ- zh4wI%^hnIN{R&~?^1!1lBW5qd<$T|pJql)R?PZzn{o>Rs5ZxP0(|15pUHZ@~EYiZq z-h_JG*Y3uo;(;soo=L$7PbrM}6}gn^W3x`0@JsUk*AE?U!aOv>n$6hh3{~-rJ&0c$ z;p!VP*^A*hJ`#OAcCaI-`$AsIv&!iHm^*>V_>~;b^_a*Pc3k3#MHh&L7raUoN{;(U zl_~NOH+qaH77I6Cj4AOLH<_F$^%0WJ&Xl=|n~g=3D^?y}MqMaY;4{Wp`e^9S-dv8^ z_jYWc-l)~uNbeKRsNvEG`sD~)^oTvem{QAV*UD&K%Yaqsn3~7vi^tJ-kJ%feW1p4A zTa0?;!^YE8+UC+K_K(JykS12x`$4J`fa3AVv2la432E|)1lRWSw9%8W2_mD(iPhSk zFzUWC+<{}lp)yE5ITI`m7h#PMkwf~4oC-A@7k!)%GhGK8n+n9iSKY>l=Ltc^W+Jk~ zg@hB5ttk^cQBoP>(s&Zm8AB+Z7@4qhNw68&)F2$N6UJvJ#N1B|yyHp?YZQVUxGZ*r zqHDTS>;p4K{SQX?{A-Mo6u8pHgtBYJ;^P!*Yk9&a44T-G=i`jJPmmXCg!*dFwc9C- zPavHERx_KwTdlJ}mipTy~7Pv+z8h7;R+lG_unR_ATT z7E~w}I=L6f9p}w2mWrPinZ%ZPJtlm``-@ISebyGTPlg*fhVqTG3&)pS#us}j?1#u! zYtmL98dh7^#!9dkf=`CV%BLr=nSM0mRkTB@U=)Nk<15YS6VvJA?d{8RCnF0S-fKgj*49mK)<;&BcyQNe)K;Fgt&5az*osele;N<4+YCZo8=%;D~OJo;aYgdwTo9Dv<(FbNH4SKbJ z-RtaKw4UAN^5#JE?>r&jFS5U@b>Nyz)R}frRG)*W9h#x(ELP`xT$g(phIu$FL~?eb(_e96dzts0)cR`YAx9bPY53OlanKgYq0G zd5-j~W~;P=?i@r^1>jc(d;9>9R)vWf?^L@NzcM?O=wSTnH5 zXB4IDD^1l+pBJFw?%<@E)TRu}AsguuK;IIVfF8T$lz$v)!ZE)>(-Z-ABfop|)3oLY z-?-$N7Z?x9?pMvHu>>xjV^n0l1a+4WT`E@SUXB;CP{F{TzG6xea%;-!u}m@&9Z|-l z&`HFQe8;8rg4w46@31c3%=a|K+Fu^7F!VOD?V5ABR%R}g9Bf_l@vY~uwoRvkVm(jy z6D4daGl|)8ylTG^*H?NqMm|($+PF8b$;CXJFKo{JKfJwFRMmgKg-fhONi15rI|M|e zyE_GGkdg-J?vUF(|>=?-C?rT;hge)k?{k8#Gi^}-8>jNg~}na_Oa6}m`# zxxkI54!@uzX~G`mM-+TQ<{u{kZ_L4%4v4?LJs&cG=V@ogs}4$B)H&m?Q9UDapFJ;F zs+ymU*Iz6#?oUa;mjd8xJ@BnJIHg+Qz5~2A|K=AO2>jnfR_Ijn|01&b+sx#@iLCyh zuK6GT1$9-(779eA(wsR^%9l!Dwn`xt>ME2ikT3X+y5`BFi4zTA|5UFs=#8Xen6iMN zt`L#ctLw6^2CLoQGn3YRjgO$dwm+z=`9l|q_QJVAk4#g^rvnFD!-3zZ>tXVx@kq+M z8jse+E7OV8JI5CZ?(5Gp#jmag8`N$r7QU0ECA#Y@-&z%mAy4)Fo|#Nj)OH5yLS`nD zquMUT*H#|Bglmtp1?#SI+anY(D0i%_J~{iX{D`7i*Yz3=DNG$hMB5$P42aS3U#AxVP^eJl z(BJ>3t{OEjx)06%!eBS!%G0Nx+0rKdNk; z_v$AYE-GQ9i1L2PwU~EN$O#`Gj_U)j$@3rFd2jX$zZrkY3%@!HU7~KZ`P6uOkDJPQ zPux1A^#G*Z@B!fb7wyr1m|pxB?a{wYFXBW>Y)8N^hHOW|91c)K^1n!aiWVM{G>Ty% z4$TjfW*#(*lP^Iqj;EEEDhU0cD`l9d`E|fJiPQ`6Td-l|ph1d>d5B5s>xw~>2%Cxn zlXT26DboO_wFmu7H;w_*tXD6HW`2ICZu&VvI3Z@au*^edK9SUWD|tj>(sb^Ly3)Ge z{DnFXv$M?Iw2N{d#m$QwGrpEsmQ*Z#EUhpB!$DfVcCJ0;jl@D0?lq_MA1m91;Vi2n zQDhiD^%0k9R1cB&S=I#cy<#$$l4sDUn}Z3ms`s)QX3|^pDphaX+7h#Fa>@u});X$R zP;WWy0@}1%kIKB!x?L;%(Ega)Ytv!!{EFox3>xEyA8`JGwq3eRBP{Bu0%dAF7@x%K zdOzrdv#R0SFsk(vivaBhWFus&m8rABRfp)&dhLgW+K{gl*v89LMme?vzKn64jJ%cO z2mex@5N;54m}DS|aFCXMDIgtTd;ozT{}A?Om?4VE9SYX%QzQ z!b#M;@>y}&`aQsT<<+R1v#`^8M&6qDE_n34kQaD0Z({?aU46rk;-z8B<84L5wzG(Q z!;YQGY5lInr^xy}<1fth`+ELsbqAVpUUi46dBk-`@-^+XKP9_$YmbGeUusTxw=-)_ zIj;C?&RAgAtIz2%y{j)MDM+d>iQjg7zj`gA_x&0}1zdH5{3)yI7Ve8c)g6@odgc9N ztas(ZbuLNe<7stA#nVBTUd6AiDRBAo%2rnS%iN{S{eN$+HjK)C^)*Hu6(C0VQI#Cz z_oJ9plsV*wqm>DN<7Oy=!~iCWS6~2@YBHkOeH40$qCf5N;420VVcfCkAdXftWR)Tj z%*bBv2^hl~a6l0wIq`4^N3lYzhYJp6{SK`Uw1H;2BqTKm{0coK$k_;DXWWy39qNR` z!hjO<6FSF2RD{DVYs7xDw~qq_)yWj!*;m$xE>MUtN_VORpu3&$(Ts;DOg8ZUnQZ5a zpA5jow?m?h&SrGBj4G5zM-?H)@Wc1fGH_{IKUwrR_Ca|F4I=(WZ+zc?4C^6!w!X7cqAp7Q30_X ze$?>8)z^7m4Gsw9D&RD2xNfPz*+M?TlMpArAFjf^UMtU0_g3zVU4^%wU4doxZSuB< z2|rCeSi$QSGacMI#8<~Y!GMjG2#swf{I-6ggfKo4WmiMA@KRA*{`wPcgtVAzic*YY zEggZFvSb*$GFig41kJ9Jv~#U8DP(iyjgGX84f`}%FP8CpYz2AsdX?N62XTRS$_hFh zsyTpn;u1QBO1`l(xbG6AWQ@&KKXa&M@z#l{dn&8h)X%0Hsmbfie$=qJRHKWiGc_%j z`lwPnM_+(#W|JYMr2t7jdmYW39i?<+YUi1DXU)8grSv4&HQoR@J_oBw8Hm&_uuvs_ zjuMmlB*6ZWP3-nFlctCXJHhC%^k_|%RScmxL&{|CCW%HBwONBnv|I&;75rkSML&mD znKuytSocoL`eTb$mR45Ho!X)`ZvS<)UTpRwvps53RDm(K%|A_GG2Si2K zK>B5})6CEGqbG&V@7ZhXhP`*R^1K^fe|fy? zfI_YEftTHYA-L~EkgxK?T}k17ng8*Og7D8vFAxe5${nWnKYWu&qk?|-r<2bOgz&Nc z^Nm!Qv@a1(q4V$Gq@$HVI}(Kiki!yYk?;0!)}L=uZSiz8PL_E(AdbF1(j))x-=vO9 zilXCnx+qonA(2LovgL+9-y~c^PQOQf6KstS8*R z4~f5{43iC){%f(pnV%k_;+wXUdbP(J=O{MMOqBcAVgsq#_W!up@c&N5ClCZuh4iQI zuYRtNcR*Bpy<&nNJ)v+k^kEP^jQo7yI1cn}k=+vX#Q{$5$V9H`;y3*Xx70U0?MVHz|_p=0_~_#`$0n z0wQ{}42t7+yut2$lmO8U1xSEjDrLe5h6-*n9FYrgQWuseK`@jmh!QPcwU0}OALX+Y zCDMVrvn6(1CK zO{(Xbr)R3ti6tTBf6lkbKsqY5eFxS&62O$lQPt&I?NTgBv0h{la>D;Sm0yK3OyGz4 z$oW&SaVy_KcgPGxUYXkGG%JYMj1*H(f5{=GjfAA|aITsse$1O9iK;ym=dG-M8#rt;V2$e;eS1lR> z=%MDm2vrQv#VVBSrPsXpsxgv_(=gD>>~;~RkDT|qU$T!a^&;F%HV=P(ppT>Zf;7@S zk8od7BGm{%%5fx*_->$||M4Qq2RWYbXq(vUtth(sQz z-c?Nvr5Hq3@-8+9r6&pb&{slv5OO00K*e>!k%Tm15?l-0po9jmX>5$7X(Oag5Snx- zlDC9VmXYlqe-mnV#esS)j54DwMNcM7<7mB`7S|ajIXDVp&t^>C~m`Z=9pW^3FrknUB|nf{;FX@KSSN&_oeXm|h9*i+m8RV{sqr9%@zz z3(AC`x(faNil8I5W!I#T+-|6{Th?2bSdOCp9$5Re!@_9crG3LB+`G_(EC8jr3tiW2`!{ffpCGQK<%?)EPKm0>X9aAJ)bx zhR>CN!6;Gzw@{c3I<5+@9=&vrkr0$BFA<=hY&EfesgN-PkHiUrMO&g|6fJS5eBeNj ztnKU$>g<9rc0z)%O9`R5lWMO;8_F%iOa}A|9W>zlxG3QLD}-B-&3v+U-in>J=eGQ`c0bn8euuI}ono~J z1p|NW`nG^=kW)l?E*yFEpj-c+brARf=#M-WE>YuETwA(l!A%7X$ zqL9al+kPLQhPvr+k-SSC)ih$N+wsWR+WoKO<-hi+6#l>IQ?dV_`c$sJRKoruFSn49 zls9CX?f+rU@a-m_}U&0}pBXUg$h7$(Lb?ujF4vGrcpt@EtTQ{`iB6=-WrBx(DY{7T18jl;oo#JTcNlpjl=4l@7}u}3wn-2Q^kclW zlix>s01e(|3QKnuZur&k&Bge?;=79ucop+a$$F9Xx3>2M8&8WD0L&5Mb-72=B_j?< zat5A=ck5HwL$tjflfAi`=HDg%q3!+qy3^m(0T6AkqBz}d{C`mg2-CyqGM+=Pu%Ji1FvJ+miPG-5uMCw7%)1OW)yxh$4j@o4iqw3wX7M+iqY%Y1 zw1l*i79KHM1_>z20d(8z@BMf}$JFsfBJQXCiXD?(7A&&V+D3S00ODF3A&>BrYN^PN@f`=V zm&CZ}asqQy(V!HNQZ_My=17PyftQn0i{a|eER-Rr zS;e%l>E%aFL~0souDtusce@H7!9CFrKM1Zzo0_Cz$ymCgHd+})xFAFyA-JKASw*0W zxZ4q%mIFb3y~~)`K;T-n9@%`pYUl}nvvZ~CnkMsbq-3QPJ-JUEpTnO7Fa4CAu8#F6 zOi5ytHTu&Y;3{cbW6dk;-TzBv^v8D4^~J;|qsy8$h92v(UJ9)G4*?P+&*$S5J@12G zyN`3<$Ga^})NM%2@Sm3uJjDj?rU^K}>dxALU&Owb(SxDNVqwpdaG!zaKJ>06f036} zl&B$pk(bivzJDh#Q>A=~kvA=M6Zp_mt2puv&;$sbyKo|_oS4JOoShoz@s>b=??-aX z1&g|gBLg9R-duIC4{Z;*NnnV$EV){AQ4ckLV5od}u5vbQ?;AW@aaCT@z-mV^Jmc?Q zwU5Yy+ZQ?&W4=opu1w%X6++`XpeYPhXU5JQv@PoZ%)~H~C8mnVxc$)iXP37L!7kBE z_=8`FHYxmnxd@9BOF_oNjbI%q2g!xlxW33_puTOYXCSjCOA;M(04WbbNy~eQY{85; zP@{`nhDmy+;*h<<9=eOSPn0V$wmJ@x)?SlJ8j#9*a{-V&U9nG=qTh3h3>}d`txmC5 z{^pd7D5t?+lWGS)P_26HtE^aORq)e?<<>FJ7>B7^Ui1{#Ay zbM;O$WulKq6>~>(%}aGTwnVYoa>g@+JK32&C}sNaG8%4cH<_;3Q@S#BDzxW9DAu;! zgw#C2JQ}yv$)jZ^?}r!1+-_^KQOeCEUTMBN&p`;LGBxzCT@db#hchffC1;<3Apqxp zNG}B{TOi(OvXUCs6PBD#;` zuisx=pE9gvzyygqkhjQxZL9>&}{vJW03)Wk|0d-I7$3c+@T0-P;Ds-G^94-cqC_P8C|+3tYYQ61@3FWzID_OmB}8GB7TevZU@MYL<2hoy{wc zi2ln&J-n^+4_q4N)?W}icuQF4xs+Viz94|Li-mBT6+?8dVM+LkrPP}hlGmz#6^*bMc3p$*nGlGUPQlk&{ zq>tsD4>7hc1}8d}q!Z5lhs7jcd>8l5{V!b&CL{=c@nn7!W)4)6rZgdbXy1Kx=6wkR zeOV4bY#NU2jY=FY{>YO)GF5=L3w~d3{rDR}aI^u0k^!ks0a}HAZ|VWr6UfN=08K>) zo+2f_g8+n>0C7b$eFP+Rvo9YTRmBGa>0tt?zX4Si(5x5GG+BbE)Pr8v1)438o0}OT zZv=I)pxpQXH75W}62Z8}!Opa=T{R44^a0P=Xpdl0WW#89E0z%1WLr}Rgmv!%Ec_(y z12;?xXK0HMD;ZiN;g)QMiDd>5)rT|jLGYFQn&@_1dnQ7L&j% zY;5*RfQw2}NGR8WL-)O6&};18*L?jUfe9gYqb|YYwBeHo_R}FjvywscA@++0SnC7a z%Z*`N^I=&MVGKojs|a5`c_gLC`BAbU@#$&pRCI(OpkSCUa%*Os>~#w&?YWerg* z7m-A!*1M9?^ajz`rq(c@{g4hZkPmH9U7gXJFfc#cVjnu=y1E_QN7)BOp&`Z)HARyk z#(LbrP)Nm6Eqc&|#`+h=UM|E^k?A!f#L0sjA_eCo`4_Yn1LL?peub5at28wS+r|k> z=?@+#4ogOeH-$hVNuEu=St~!pWZ&W>@6e>6zq3}+i^&Xw zvG$)m?nJHLM5m-S{bsE)A=5~Bskwi#R<5b(uGXd>Q|_EoR&P_Q@Y1Te(rQ1Z)w`xO zrld7DrPXn1_sk~HnyIoX!tNMpRen_Dn1CI0wQ2h(&qpdh3zx<_0Xy?CW6m{WAths} zDPtuiqrF&Bt~kOFysOR|4ZH4|iGZCMRLB5yQSh#Vp%H}L`AE(%0ezVQD|oJcfk%FS ztRWnn1!%|Waz|I0s;Fcri zlMBMn>7;WE>&nD2P;JnM7>$9D2G?3Gj665uCgksNfVp5H_FzHuAcYai1}J6&SZ!e- z_GE~8p$+190HGE_w9Wdtd{r<*Ae5nO=mveXH7xS<96%7ndjK)_MWa~<6o?(AVSII# zD#^zJUuZ(4%L=4uhS{htq%dOH1s^(Lxumr(Aj0NOcy$meel}pp2PIArRwNshstdkk z2=)RBh?$M5tVw>UUjpgrEmRdkwv~^dfSZstWJSPGHYz*BA(34gY#x&!l@r#?f?1>) zXN#m*f_zz(#Q;K6%Ld%CAmUEIY7W6SfQlfq$ob|aG>}(#HeAmTY};47I8YY5BH&gL zaFmUAmkn4`1>pOj5OxD+N>J~z;j}@h1F2+^y>$d7HAD1CBTHq-VWywB6)A7Bo+%+4^T0cY97U)~hbQv$ zuUgOd*^N{UEhY7(gY~0FWn>!i4haedvyG_0rl7D0-n94DuWIAcn$|R$E^Df6N}IBW z6VhK{r3u@+mA; zXtqe`c5m(BY(5We9Z75HSoY>xZk^In8-LXn=-BoTbwEXa>jwChcz7z?HbMLA9R9tu z_M?{e>Mn_)z3q}}?UL59mnQV> z2{l057A6#LWzZC%3W=9ABFD!UBxwzhM*9TfW>n?Lk(lr5(J*|koQ~V=$H>kO!dL8e zdD|yv-wW3NfHl=;dm%OQmMb=vE7E|*7O^Q|E*m|-AAN44zWmDccC z*U&}Ku(reStNh`^zTpL!kxlB6R^gF(laUs`k&$mB-?~P+_C^X|MjNR|Yj=hbL`SPk zMgww3^ZQ0!w?_9MF86G#d=HodOX)`}Keu#NIcUp>d z;tToohwpF#ZKT2xSfWI7W_~mH)iV;4Gd6}ZYS6RDN&PVMawz$MA8)m18D&QYSHSq-0>GSy5-mv1dhK=IBu8@v#?p zViw9#7nie@T49#+QCDWLeb>RPL7}emWNDDYY_bvy<(myF|I)^X*>OVEV#U(SgxSAJ zHHh+Nm0JYPd$l7>NPTPlc#U`^pOg zg_#OXo)+C21Cy8vJCg=i7XuHKiXf7PNFIaag_6vRhJqi1>Xed3o`#MYgCSD}I&+S< zZO({Dkcpq>4KW5Q@sGD_bMIc}-lGa~G1Ktq;=)$gNxY!*wNr|{P|KX6%T-V+oKh>T zp{r!d3J}i=@z0Bl38<}6tB;|7jHEOeqyE&6ZhR{HIcX37`<}S{o&^MqOW(ne*nbVZ zAHK4u*DhdRLG5zd<usMRzJYDI}o|t{|s}8MRq7Gd}wBP zi0OMMlzV8}by&W0@JM}xzIiAJb7W$8#JRHTwYqbc`f)%x8G4-*t?*dCUuQqDOv$AaufGc%tif5_WtP zj`DM_`$T8!1TN>qWMu!V{7yLP!jPBM-rCsNnB|2$=zyPMXN>A5a{P+f@+q_bInn~` zb;cHZrp0y+jb?#>cG|;qx~8(5jB$Y>aDgd6^{R6IXlx!iY891)53Z65x8ne>a)Cf$ zgDC39iS+_vB|bV>nv86U?o00kSP)d4O-7nMb+{%BP~bP&f@b%F##cb6V*z1DvyqH% zlS5q-0D~~O(&tAPoG|OpN@?i;KE$o@Yt;23BkY$!I*esB9vn> zp+p)R-jL=%uM^zPS)uQEOzfoJ+W$JiXPuPw9EY~T>eLlR)*W{pA*>K3)9nDc*xcWz zk41v-qe&iO1s>w{9un8FsytvUya^-0SQ2`;p3?RgbDHg-5i~~oN1uuzb7BFV48@m_hTda9 zM4vN?POZXdvDyG838D_j|ENNmq{5mI`pa}`mnjX>rxF{$q!@Hd zI~(h5q^kYk;H|->6fHC?t2BLS?WHv1b{d-ubIUbE+k3GE@53_8u0rKF2GRC%EIVAu zac<8@B69t#y^`n6TVaD}d#|q)1YxmfxX9t(T`Tg4ViJL1#0ea)l_V(=9F(OQ>aLY# z-_Am`y*xM9DvBZyZLhM-yBk$i6^OR?1Bqu*gB;jERY{Ws`Co4D04M|~5179Ha(jQt z%c=U`_++VQUU_BNkI^~OfsA@X|8jf3${x<-_x%T-oESiQu`m0M^BwI!`e384t^eK! z<0xedheFt6_=``ryX0|u^QRBi#aG%us^|NNqZ=SyZT*{1W~k>b-qH*3U}QUPZv5Q` zTjv&Q(Wd$1Wwt8Ba^j>DD!6$11{fL397f&CPLnxe7opZ}vMPw*{9K;DP< z-^}tY*pLCfezMG5(V>7gv+Zacp<=}dN^()-4_Zh@yD|Df#(u(5^jwOc#1P{5J_Dga zDMFv0uc)Qs@hEo%Og-^FB|(O3(L>yV=*$dBPp_i5fp!MbIWV6=JA(J|Sf9ld7qAbq zlgSek4MI`#&CFy z!tApbN@Bu>p7(KuhDG-yL7S1Iy_|mq^&nvTGD*&ot6m5KzA(OZ9>fqe5ap~e+twt) zJD#YA!CsE8LVQQ?N#k*>j9u=%UNLFKfl%A^jI>b3cfU*Q05(Ym=e{#R27>tS;+Ghd zT8pZM8LErQX@Da+8wuWuB?ko{+n8XTRbJoX5V+PjjD3#DMQMw9;>_3#t-?h+^v$VD z@TXIzyAY~()-hoT0u@}!s0YspyJ^8|Nw7q}RI(3>mMyXcOAAk%zh`zZyL%_|U_Bi- zZqwvt2=ATpd}{zR^SYrZVEsBDm{&Kt9O<4no*I?NB0HE;*IS#LGp&5S1u37cUi{fE z+qt82eK`2C@A)|CkGAr7{CmmJ7EkWal3|BW)XAHe=a=E|r!DxlFk}|Uk|D`BMl#(! zYZVhb~S<-%ExYKb;DGl+>2}ZOKp=MZ78iQUZfl8R)7EFaNV-n8(qLzhq0z z?H>Z?^q+6h!d{j!N!9jVq%eeR+BkF=0DaF*w_#)X| z0x>S>gLvhHs_+8FH;4lt;Pw-0MhcjnKT3R9s^)6kDPRe1lF@sy=lVhUjV;Ml`V+AV z=b#aUPv(*`^`iA|91+GT8XUF8nMxh&HoTbHj&aeLP~AvYcBo`Bu_68f8M~}}0tP+#e7c-mCEJ`g zKze;>zWwp05|XFmgbU7>!PN0wa&MkB<1h4a-+qs_D%F0aC0{NLFSwyncJ`X!PoD0b zVX4zC!Z*4XS(7bSZm!PoVG`d-;9D<|`A3=MB{2V7T>>AYyCF}@;?iwHrz3;);OE7g z^AAq#D3IH`%-p71P2&;Q>Cf2Vl_UE3rquvz2ewkhR(qk!o~jC$lrSA@b6oC~HWJ7F zVcprpq||-BCBEk+Snk$I?r79*$b|HEXr(Q<)Q=AH~ zF}eEv;9K2`*Bkn~`U+UjM}GVjj&hL0oe z*tAPMEa6ea6sP6?@yQKMyszxB{>aa&dJhHp)l zz!$-Ht=5%?X`S8s{Y)ysH@$`%49php}j8EuF!?uU>lUw zQiGxZ=-ECYI+OFT#cpt^p>V(08+*4E0)C!u{b@C40pIeS>it7|+v=IrKHdf+u6^GF z{+EIODva5M!s{G}Bt3r2(fyGBO673f zz-=|1SU*=Hn+?XHIReBE#Jvoy_<$@iCbVA z->E}G)lw9bjkg8fzB}&E&F2^Anx(en7LKYlit)RipDR~i(OPR4QLP)k@R!FWwd~@K zpY<&h&e#a>Z4H+{&W7pE`+^Ed_RS@j9< zUluaz&rPq~SOvEoFLFOKtAlR=W4z#i_Z`pW&Lqo24ec9J3NF)0-sStj$CsvyzfO1s ztR_vJ&nS4jKhEj#+)2E9DD;0(fk<)_$k~vzeFC7p5U6_)z`+^78N#o25TL1uHmZnLn+?=cbD)~{iNy|n zEe^4g2h0MIG$)WpC(yzn;X8Ok&1=Ef&dE1c-KU%?gj_rX!8Byf7!b~hGOdUf3>E6B zW+AZPYUT9R<|d#7D{!7Q@U2R)2N20B8wkncA4rkg`#isecqJUTf9m~Oyz_MqI*g4X zti}vL9|-T_qwIO`ArHqC8zEeUCcvLFc-KGNV-`up2j~f6t=tTX+|r0Ve{Wgy!8zH?Hnq_qol{p57%{`?ybbwEA~+Z}hPj)DLUEa!Q-v3a zhR?=C64XT2-9#BtS;$EG!!$V&EO-$OxFURv>86e#b&8ToiZW=3g2s)u!Hy+IbkR?i z2O_%O+{Vz-hRQ6&{s2qH>Y2v6;>H;b*t3UzVEh=BWEPPU60t)OlZ72WZ5D?c8iTwj z%gg0K`(S~T5&$TSbAF#t5u;mqXCpA^qrT{Wf|KxVCptk%Q|+^hbfa0Vql*cmv>6u> zD3nEdF^GR4Ab-z#yWS4J*oYr7-u5 z%*0ttqn$i!2Yu@oQ$3vY`0q_yQp}57BPf&oscHQiT~nF|)2dzlhoyo?7n3;-!^lmI z7=q0E_R_M8lQF<9X|AC$J&VSxi)kC7k%@!4k;NGbNh$WX89$|>x444n7EQYbtp@3W zXHz2P={%vGeKSLoH$SF@(Iq1_Td9esfLyZRT>Vbevdiz%2=US}@bw?(vR;Z)M;<=u z0;BLWg9whY4P3GeeGpu;fs9MJ%#XP&_<3yHdF+~b9Bz4>sd?N-x#H(&vYk0YhdFbL z;E zt;koiC=|R@6!};bjb9wgUHrAAK#M*f>PWni8ZJE*XqYXRjbD<>U6TFP;v16kH>4sX z@shHol8VQYD*V#wqmo3oVw~nw(IH@Avz&HkXNu~Y_IApV7p$kCN$@><*g3eGAXXmDN%;V`DTD`tBVZuia2Yk`WjKq~-HUJC|K;LH_)g;n1_K>h^W zSq&#sfe@y9fR74+&ExdZ3ZZ}oP(X;pxd9?%t^$f#pe=yV8nTf^Ucm?HBRF>g)qPM3 zL1;%H)Xi+vi)>_l9>jAbpdJCLv=7Q16mXjz@v#%W0iu)X1X>RRH>x zDG;r8epP$PQcGchMvzwhfCRh+p$c0dKs1XUQ1wHd^%o%JLQ*tyqef{Enuv_zT{bG+ zaGhwWcu;mNL`%Jp4Yx^(rf3T(LRPw3(TRP0M(JojJ?`+Wuvr!)e zzc&cB>Xf3|TAcf9@)sB0YKZ*>V>N6 zllc0K<;qQ#nzzI9ah+9#id`019RRhiml8M%_ijb@YG?O$qE@fur2^~g7SUF5nd3^l zCvgvb1SQedR+%1|V{y^&2C?+^+t6&SU%fn`@!N6b5EU=V=L)Eh3X|48Q^vk6kv>aE z>$0|C47zIrrr$ojKTNcL)S!ReuiuSuAUvRtxv0<2V<4obe+0bQ-w!ho@@v33e_;5> zK;PCtw8x;A=zy}>K=R69TmE2WkzywAkQ?P-x6oj%GT?K7I{(U0#jl|%!r^M(;actC zdXM48^x@{#;ntPm_LZTT62clD0enS(wC%`1`p8i0$jAx=%^jH_9G&7FozWhh^B7%7 zA00ym^7;(9Y>ql52tqu((ClM7tpa-{P*eO z+co3A2>CO&$DcO^!Il`%Iuk8a69zdGu&eyFd3?s~yfmQ6qp%4Crgs3VNs68c{8e7U z)ycuDNm4%EuVRxJhEo)tct5r&n%LeF1Wmofh;*vFw*YtX+WDiic$N|=iKBE2ZNyFk zplA9%QSwidqQjAQm635V&8$x`)Y=QG!mz%&M(d3uVQZUijGHt|L=P86mp7#R-Zev_ z#6*4~fUY{XNHzz5L8@s*qDMh{S|bd!(@-a(Hea2hE5}5qoCfx=IulXh`0`n=kbS?I z6FQ;0y_|Q=c>gwM0oHGh0%bAmnLCzmu~=m>k%>DwV==8}G5whn;-U$|T*~F+%-30R z{IXP3&Qa2~q&mG+(Z*gyw9G8GJkGvc-^SXQu?%0ctjapsdg6f&v+^UOuQx-g-E*a? z1|8oAhJpoVavHke6Y;d%6#fMPf7uEp8DOprJp!Cf{2DY9>4|Nay-FOjB0dQZ%L)e+ zTG`X-#~_DZ@`q`M<==i@13|-p1=pMW*Mxf48@Jaffg3dx8{?uI)t@%{12*C#)}?=J zROD@R05+3OHl&0$izqhh12$8wHYI;-=HzXb1GX|wHYJ3%QYp6b0=BZPw#0vIMdod# z0Jif^w#0R25f(`+7|t>?U%Rh3)m?>*%lGnaiG{S3D_yK+7bS-^Eq!v4zOE! zvLh(8t3t8+K47=nYM1ZFu5{im2Vk%6WS2*1kBwsQUBDh;-rn1uy=JRDQ{8p@3QXHb z7Heh^=e1?WOiXv{H?G7Y-nwsmQAPYwA@`&;;oujh&@tgKFUE*W;i&c%PcO{4mzf0X zLx;@6cXo$07>Bewhbb^eRFp?=M-SNzj{Jp>DD#hOBaVvO5361^^_lJb!?_K(ji3$~?m%!N3|P!H5zd5Lm#oK_?k6 zAd5n$@b01VMyIKqrprQS@Xlr$M}Gs>n`G@kf4koMt`hydUJBQntJmvS^x!K&J#-#$ zwg}1fv*$JQn`>zjbg}ho+{yzQwK42)%s;O%nIhK@6p6AkT(X#+H4$5(exZV z8FaWe)}e1@F5d6o81$% z!r%C0-Xd}GL4P=sOwq-u-M{n6)FS@$Ho`lw=$*pM4Ip{~+US1y-+Z#C8G^Aw%+U*x z@n0(5I%_JaPlfmu2EBbd7%CgxHHc?ajBIl!T8(zQUjs1Mw!3PHhsSo8?{;-!U7sN; zUI?Gu8jAjgShkK<7{M9(?KkbkKYX%H5N8U6PtFmGqSpDHrx&YLnyfDU=97)bzkz@A z$tz8cXM4Z->++AM*dtc{k>njU5WRt9EXH zmkC^9%vCfcF@lpQB?*dNl&JvPgzKr5e`Tj(AX&NBf6K~IcmDURd@v3XFIQVbad!x^ z5B$%p9P7ZIMhKt(AB&`XFa?NYbo~YLf3eeQrN1C%;lHudxwVBLf9^-&wS^<$xg}6Nx1;?xr$&f{WDux`f`7_+SoPa1tQEM-Vz>1b-{dy_k6HNjo3s^1 z5bbCGT%rGqSs22p>9W7O`MFR7+45{_`G;Bf*AL)9TB_c7JD+^*>Gp6EvYx(Yz#Vuu zYrF$23J)%Fj`jimNP_Ye+HJ`4M&?Kc`N0ZhX9mDC(Z&TJm>g^bU5v8C1Ys%1r-*_$ zAO&M0Nz|?OZ7(sD}QiJZGH%T+4Iw$J79(h9k9~jKYW{jI8SD9jX^SuYY?!qE=m4 zLznFt?HxWuT3!y)7M5KOF*Y2tMT)jF*0__5LpU|g{j#eu+N6=IaY1;d>j_bsk6$Jw zY09tv;?$xZkZ)!(rQ~jAqd)%feEf}5TQG_$0|>b#GOc!)H=O+Od^B|1&+}ogI_;O= zLEPKmRi-ts-=2>Eo5Jz%K$;4O=i~k9{cq0)mv_o%{okIC%m?RGXNcz`Q(;XLT|55Y zJRep4%zxw5Dz#|1jT9l%$bYOq=RLUp#roszk=^1C7kIt!J1bvtKI6Na)q*7AJHM?z zyDB7Jrt^7!tUpohk3VJR+JF7A{`5`j37q|U1+o5c3Y-w0nA}nPvHt8moOwq|>BG`U z3L)`>`~qV1f$s-|UUU>W(Rnt8*%y9bq*nR725caz4+vj++Hz|S;~~xpLbCGz&Z*fg z2yeBq1bR+xP+I@^Pn=rLK?^zDS%_q77+}^m%re;y|DT-NKeKYy2v3kdt}lEa%bam| zLvkNmh%ce2{w9h0jtI#!XA&#YC%mD_E;7s}N%8vLm>#oU0l^=Cvhr-?0$K^FA>}_= zc>z7oLloVCT~tEtjwZ|E*AHcviH*Or@}}Vr?Nedjk0TI*#bvZEFOvskzp*JC%193> zv-E2J-tv^w;lNBBN0wzTg=FP!({b|v6AtK_Q5(pX=OD5vx zA!lcfl@a7X%7;y*?0Y0_is6{prQ*-R_IV_~TA9@MzMs0xZPwe!5mVEQo+GOSCr{Ub z^yAIZhjMdJmHZE6d8$&4k)sJe(o8@*13>pX0inT9!mBFJwb>XMf zJLC!!xPLF&&8H&@L7JkC{d4Ias;Is`{TZl1r1busWu=h6MEaS*k^uI6y8=U z_=|pt6&!zCTjXeJT)Z?rUSD7NmCwRbYdC0WX|5JwwA|XCMs01(0kiv=$(CnWpx}Kfba=5}2W!+|R-Ch!hpiK|U{}QM zUP!HLsa5zHV#1)>hbBlFVTn8p_^PqVrW_k72;&XF;;^xm%IZ>h<5fw`*(-arD$owQ zfI)_*#u45Yd*yI-ej;jD(hDml=);_2jn`Hwm929;T8iUJn_sew%jQ_)QpN&I3N(5{ z58i!g8qWT8nN`TEFo(l675hSE&ESb8cBL{+QDKwKXSK`VJvD(j^0rvY@JL01Yb2d{ zHx&NlNL|5griNO&-0Imxkl|*sy!;ET!K&a#cpl2``gh6bmdEN8?lnWYx0Q=emIiwG z(|MwG@hOAmx+1ReBPQ83EhjN%m@}ir6;39FD2KWXj?1Vf&Iau$r?yY7JwELm^+FNW z9yIFfPbL+{X)88LUG7~Gr>0F-vgdwbsmloKp)Ex8*4kR`^{`|06)5f7;oC+KM5N^8yFTicz z7$FNjQB%Ox4RW{zlzYO5%f-(Q5=`!zNU>BWC>CIWbk|I*;*;N0taj+&bl{y}g ze13vO$c?3(UgHgqA{ZXU?2b|Z25FMBymbutOiSj@rR_&<@0UG^BYlkfjvk|ZMep&> zM|v&)3^~K9Jj>y3_sV>Ka&s??O@(f6toCDEUumBq7Q5=lAFbsKP{=={@qQ-j_m}Ti zC>zC7kq;2_cMpVLI9q`9iOIcmumVXSLHgP+;hBMSnBGcqZfWO%X}@p(wNtuQgu7|v^$vJH5qLivw0iv!sYvk*OSk$l zU{W*cE+=v>A*!z~%9TH2ddCxWAb_|?oOIWnoX!4-JhI_DD#g@*AxWG`!Tm9|a$itP zV{Xirwz(~Q)*7?X|6`(h#-=B4(Mvu-eI?D%`uf=J+1PYbeO!t!&AeYSOuXY&{Pw11JC@n|A|)v{U5?&(_g%ofvtik6)DM%`+bN^2gv5iLgwyWC$^Pi^ z+?ZQOJ#x9E5Q`)P(YrMv2_5}|?@ZFZzjJ00;6BJTKk4IPL9(kwXxSV98#~lPI|11# zJdG{M%_RjpIz^Ke;OnC7RR}?}*E?MR-3oD9EX^GpQbo$sR0;Eo>>_ zaqkdHL7iF!zxmzNqr9Xe7NhYSW5k}ut2UX`IUN=Ja^=$0FF(DtNK=*cjiF%1`3=FP zOUBKd>CJA&o^wh~ZbrdwB70nhK~iR!plorb=h(~#l*)J{#SF?xOn8Tv4IrCQD4SU~ zo5eL7l$_1pl>J9m&Pe%2cFrBSDXo(zK${!=^{ArIe2&;{js#_{q)@K3Zmz6r?r&Ln zagH{3&Ij!rK1vbiV*;~d0&TlI?c_Y&raZm*yf?1$hVyyvgz^ztxtVLed2ya5dse1` zbhExH&G zaiu8Q$SJ9rFVWm}jhip&R4i#%bY~4B^If zmyl9JFYDNfE<}>j<{0IWS67mo)wR;6)V~&Sn9d45tjf6#X8&@f0vzLVUVo-69FQde|A?2 zN4&{SaB^IFWl8Xn@KM0=Q$(fz#oL;0*BZiZ^jW|o(vsTorZQV9(5zjV%8})NS}jVpObr^b84S! zOP_u;pZk3u3`4+sn4XcD%XPI6El1cFp2}p_5<6Pek9XRS1*X`8_76z%>mVoVx&d}k z2JpuF)5B=)=pSR-b4tJFoMB+kIsg_bxyCC2{NFwI5dnnUhgFLd)wc}(nU&uUbx;p? ziVSz_4fneLYgW#w#d`Cs8A7 zK$L@fL{R=65tN_neZO@7jtI(cTfW~fe*bm<9f@WXMRXKRe-y)G6f12Mr*#x>Y4p*< zD1c^+P;`t~e~i>)j67|OvULm*l+!+p(bJ4GijFhukF$7;gVM&?TgN$<#-BWlbJI*b z7oFgR>re1`ObDb+2(?a#EKP_#Oi0j7N{UWO>rcvhOvvVlt_OZkV!p*@_cYc z?_udl>(Ywok3;0{4K9xDv>$skoWQCdhYvrFX_imZNHKi`&X#^$=`-B4E&~~s?nRgO zU<+{36*Pkt49^wnu>q{Mm7AZ-#y?l^Xjci9f2^fVv3`8^jA!Wt@HZh98Ul>u{vW|e zSyF+#wEsyg`@1{_?wqMn81r}_h~sa8Q)%HUosz$dIy*1#aW^cTigkdusk&(5-{mnJ zT5gPQT{YKh|Ah77p6rR${SHRb+UAOu-=9Y-LC9m;y;0>Jb41yJtud1Ln1xh1C9}oh z6nvb&mos_zMaIcUPzhPI8=#`9~CS>h$i8uI@rG&g9}l}0~6xtsD43JXn>%YSk9@& zU}w#3B#(ipT9-2V6+PH;U3TcS#Lfa@;%Fs1qH`@_6{9A9-ZOUX$>piH&YS)IpWECL(aECJ`@= zZ!7M6Ni@Q9Wvw>&{p8?mR9k_a7ZEs#E{-YkFD;BI33FXcYWxXCsvS0=ZTSO?G;5MT zLsD;6Aj&ZMXW-Q1^oLj$#O7bWpw!4>&;f` zZ-LXliDh%^mp;3d{v&XzJHlVuCtm$SGu?9BglRKz{5u$Fkc@!-56yI$@Dbf_nrV;I zX~PVcDASlC7nO8)4tU$^141*sGtnV*y`8WOJX%6cXvEMn${FSwH2O67peP`7-}^p@ z5pNWv$W4DoTG*Oin1h9_oyNcd!2(gMexhD|_3&Iu9O|xue;O1heh4J!Hb>~Pu=;KW z9Z>Lhd^y*1pyJ9vFvh&9D`ixmcqg(yWx6tI>~Wwtd~&txn@;X)Dm(%VA|!7z6nbPm zEF)m%Xs3doF`?HGCSE&hr03{yfRW!1(R+qS4|*I&vAI4#Xb>zMsAQHp(IJL+!tZ_{ zMH4WJiL@3Ssjz`cg*3WWp^W5h=f^}lXk<)IcGKKG=ZncYE0?8|5aYSn84^OAB*TmG zL=K%j;uEpF*cW6ZF^3QWcI-Y;6znc9J4oEA06G%6!jNo!BqoA}B>J2}F6Y>r&U@1P zZets$tSs*EQfM$9_fCd9>p~oD8iBz>84!~zmltu@+DgS>cN=s zYUMEoKFrq;<Xl)<;>(ic}j>cc7Uo5TSLVV{R%h-dU$C*>9JCu~4WE!k&9La(euRFE~Sa|v=0XPjb-LPF7nU9p%JrtXl$JlD%v?PI|;<{^FhdO3n9Mjdt#LH`t3$!dr9IuNrrB>_{|P%da!AQtJbiw|@IgZtS_4;ST+e}a+fpG?ExE582;jC5(Qt8^eb zQpE5NFwz+mmkG-z5PQ4|HzRO>D3}S~mH`iPy*6yFT%9~Kyv5gigMb}GRHtq$Z42cH z64GHI<4n4)6h>?V;2pj?9bp+G!Suwn*M2Od-*#k$Z^xn{YI0d$?JB;#9ZwwL&ACv8 z;_V2Qc_>F_4oU9ZIgFqO<@Yka`L_3_`F5%zqPFz?t9{*t+v$dpi{emVAhN~^s+;W3 zc=WaHlTyf}0b8wyGEImIrzCJRqOPXk)lW-AQa3wNS6Bb-r>&dtyghwKxfIeKWLulQ z8}yxWB-?t}{?XLP6Tg=2S4aNdT%yZM%n+ICt`> zp_^6tB;;-C8tPMlM*r|*D2Q6rjJ(F_dCiy4{BA%pxJu)o*2t;nd-p|5!^%D~&-AF+ z+pZK}{=v5+XX$tDYft$F`b?FrlT-S(26`K}9DbeU3qS0hKf3MxH2*#)0drU1;@ajF zt$m@}!#;b0>o8eWN#u;)2D8ug2yEoC4pr|!iq);NOqsUylgxh0=I#8_$W_|{)u9G| z{aj?kW%PIHpPni$YsgQ>ihWy-86{iRSdW}~$s7XB=?* zb^Z!d)EZso>SWe~!ud9AcAsB@A zDr@+GGv!$E9R1X_teaM>Po?ehfyQ0-Q3eMT$qNoH&WPJ@XT4ic@oH-}d^??C3ecxO zyH6-RAAY9)a6Q9+*LD3+I&8?Xib=M$O}mreh0)+;H0h;v;$;p+hsHakBz$6h?@g@z z0ju!sVIK)z5Xovi6wZyV?QQ*8%Zt^{i$ohrW-f&zPeLmIK%OOGH1}XO_hEJR9%6NX z&e+S;lW=L1JZ%6x3nD?q60X+r?2xB2hL6=s1%YM>+BcKn;VuAlGjxoJq#C0?@#LEMrhS7zRpo0N9*F zwSiWBHlX$@V4o*I<(bDLNZ{kFAlGYeOE@|7WnX|#gNI+CSb#Zc2#7GL!QTVqXEo~> zEAP)Q;22I8xIGr^c^&LV7SgU2yznVlJv#WAeE{|tHX1af2rKxKHMn#(Bqz}i^~~Lr zAyfqtTHo*AvhCk05Y#^F4r}mB>?i3hB=st!g*Jr6ZijT6hAce`M+JMP7luCW51kEi zolg|408u;l0}!^^9)80??TGZx5wo!omf$dgte~Ge5&h0ID0P4tP^5=kSIFtk$g@GKQr54_P5{Y0=0_U928ub>Psd0jV zv`L`^Y|u>Y7{Xw`k!$kK25M@!3xG?9gg%&-nJw1vbIiDLlpQRZLnjo!GFD?OHpgC& z`Xp8Y+fkB2R63YMZ~!0|48SXjJu;_J4hE`H#3}Q~jlPbfP7ET-iX_PT@*4JK3mE_I z=97tw0UHI)+hBk-8;R{4)vL#Zb`%M9{Ly)ZzNCVb4vBG?_6g2&apIn>tc|6#>DYJN7<5vQ#T5bSivWYW z#0YyC;3m}=J0*)BA&;?7n;l5RnNIt`max+3>L=(t3{KxnO1kw=-=#?8RfztG8Gh6V zoSP$Fz@{D1$@oy8-oTm;L(ae*%Md=vSO#XI40@okyJHBE#RX?#Tc!hLGYFbKjtPpy z3ew;d6XEUAJZj3in@0RE&zc8j3qH$UQp;v(@&xXAuqV@Sh7dg&q(O8wuL81RXIWQ4 zMEtHqf+2d8gE^yuIX`F8$=JxJihxp^mHHlMtxVJhkI3frSu}l`D;n zW!{HdVTpN?Ah-^ZFFTRHB~4&RZt$LPs3IZ4kYsaAta4lU3|KVvtVl$+sNqu)i#;dy z4S;kohCCUP(pV5RSHMnQ@D`j*1PMLXNlI@b%v7Ywrp(EmH>_dzd{kMCaDITsN+wQ9 zSSU*yc}gkGfk^^bpXQ^>T;Ivzl)$%2YmJLF#tQXZ%U;D7q!yKtvzENUD*IIcoKOVz zyAlqv(+oFdEh-BCpd?vQ1oRF9k1PpKC~3}aGcSaMuXG8^jtMZWh^n;9eAvI<04q?0 zDlqcDzJ^s;1jj!bD}Q`aK8jpn*qvet$8cHEZ8A@}PdKj9!+AE%b$q?SS4topZ^-TS@{z}C>vg66`4 zDfm$B@P6&hP0K7eY`i38lB$iBy~V%3i4js#*-Z3ywt4`o;eew}K)!9{ZQ=qX@WQR2 z`mS{|xt(;UEgq}AJu7IpId$KvLo}qLys)ExEXaMf6;&i+1s8Uf-;@_y&phTM-%QBW zk6~a3BX@7F8G>DJ!KPwcaj}{(fE|}s?kT<~LVZ|5!=0G!t%UP+*h4h5JKmoA#Yw(h zto2GuPf)Pva~gYl&(ZFTLsi~T6knDC0C-dk(zd`ruJKa@K7zya`B@wQU^EpWODBf9=u;Pi_Vzz|fM@6-hXpfE=v z|L4F7e{2AeS}hKvz*^tY1DI?uShxxq-v1dmftJSkFeje=jNtxf;AAs_XpWv02@aMH zGjRXsz)7xb*xk&QVQB&r)~Z9}>kRPI|2f>}9A+p&Yn4i!MKo!S2%K7{t^W#~K8nsb zEKT#9O<}_@^;>6h-dIZ%0zF%4O~a{kjAz*5^#m^a*PO*EN9p90rtn9Rj`UQx&IPxL zv}rZwT1|+TdF0n8B)5!>1Jkno6s-MEO)#MIv~uLoG-G!O^5UiWs#n$#oU_b{f+DJD zLIE1AnZr7H7^z+8?%Y(SJ+oJ@(O3a8@JzI2Y1Ak`ForZLLENGSub7Ags$wPvcK1Tg zt9cT*&O(hSK);u3@guHUo-ukB@nSYV;>HLUhDp3Z>+3=7U5oBsA&zJ)pX4CV+Tj-1 zk;x+eMxWX1K&|&)kWodG??Ejv!rCywx<`F=DM5Vo375$qACZ`&jbNsJBGAjkm^C6m z-k4h|D|G(GigHG<(9MO?MXm@z@MHizz4IurAL#ZtWn8ppHq>boNQ0_q*EEloVD^lH)y8&K~sZ9Issd_hN-hVx>d zBFzUGpq;3q5o;(arz6#8Vl;MRc1o}8Wj`$+&0NiH>N9V}5<4{uOUL zX3Nqbx;JkgZif>a^@N-UU~jPg%mCmq{>1-u%i0$3i;KWZnHH){?d^{*a=gxJw5jnG zdBXu`776|3la&Af0Yrwhv20}`7o*k|i{2Brz613a&zmmE!_oBB*bZ^k%%3uukKB>a z_kHjc0XknJs;y|ZoizbL@7S60_@6vl#;V-kV+5dy17^MXY;0LwUmuvf@`-tdoywrp zDav^*MM@{e`SVb0uw&L2eM4mQkX04!+mxi1n7pYBs)i?ur39*s#HyG*$_Ts+S;F?{ zO01UEcD>$q8>3Pjef0-Mo3L|xy27J&C>@_(*CIN_(25zfxkMV2sf=Sn=53RPcc3#mAW!ksc?R{ z{Fyex1iM;+op(lG;l|Gz)Xn-W8w41>v(LERti85eypD*RW%a!NA$+riek09v4TIiX z7Cx*Mt;AQMm_L)dDry1un1p%_yK=dq5HK) z_sixiuFC7zPR6gj_Fo8j%;6C=b?Q_YJbXO5U>MUji#wTaR-0MV;24>XZ|LS zIn-})xj5WhnECLwnk?(r9)5=%{5SHLe}x_VTRHPj*g<6` zP__aj8)2@Y(y%JzY>y~sP8Jf-?K3gRGwOF%$}`6z%9$)F4hYym&Xyq)mDJhD z2bb@9MKM;&84^I{bPe%$u6pz?giEQXFPMWRp`}_cYFZ$c_*xDnL;6?9dW}RT8EW`j z&skKcQG~qNyTH;bv@a>r$mnG25VrRTQVt#YOfqP$tjgbTb;s?3+(+KOwB23a zeds_XbGHjzXSqLi!k2SDak?<0hWKx5>Pcg<%TccP@yy(p4&d&&e;Sn{^Kjr|VtsI! ze68hi(edpz)p1tKL)2;Bs{X-g@zadQ=N~WlT0gHSa4|Zsh4TM$-iX!mblFmQXm#o5 zyt>aAcpsQv?v3OVz8iZz8}Ie$qV{(X1x7g*xI5e*4g`*b3}5|3P*jn;dj zm<*MUwSNOk@K_n(K_T1zaiX4e0=LWC<5~QOvTPMEc&602o$Dhvazb1MFn}Xbx9+Hi z(4S>guHx!w-TvyKIlQIuQ}|UpQ@32y(BbxM+BWZO16}9NT-OKqNLti&`%n0nV8ibQ zEJy*0^Fw`>Ep>{gXJ>HD_U8@1;ChbJRGuiUm@7~$mO=zrf^TO90hah5h=zEl&DX;P zwhI3y-YA4SEQl&u$zLYRGNP2p6^_`8R z7z|9cw&gYRx0r4uLNeN$DM5z|E?$n^6CO1Sgz`r9{vHlD{t$aE(ew3uv+^XA7wx59Xg9z=gzoBlR>s*#cY zm(decfz=fVV4~e%{P)pwFK?VX5DWTi^sv9|{2SBe|JCUE%;oocPaokp8)y$8;z&<6 zs$B1*rura_*H~qmfTXU-B7al0{hdMBm-f|fJ{cCC9D^lNs{NVL8;T{o=DLa3FD({+ z{3PP*DR#XGM!dtc1%eU3RkWS%e!07y;f=w* zlj-+gj-K3jOT_5OFW&oW^pyU$qvwv1DNG(QdL;erHYk4|Jyv)e0TLeP(4UP@R-uR9 ztff|C*gshuQ0OEm*AAslK_*Ae=e^DZ;D;Y>svRhZp8kJ7db&tIbPkeFejNOVPez{O z6WQOsz!J#YNGOOG5*7v$9Q*$7AL$~OA`E`Sh>r}A;15eit>cP!QE46fRPk5Dle|O$6!T(FyndvZBfq3GS@kQ3?NbKoopN*q_r2&{Yax?LWp8NXd+8|>#Mzn+= ziJTBiyJ5{`4#U^{qsDbiw84_}m`^+NS=G0k(>u9 z)>^ye2|K_iVb2v7z|y^?fb5E3tT9Y#7sh5EcP<~m2?ga9$XBcw$Y3oJMEf+m3wnHD z&T+m1?l+hB?Koh>3XLL!kBJFeNJqO3o6OHQNQ&=h5K(3Kvqp?Sla!JNg*pf9 zk|H4J&kgW2$%c}XO5(AHwJYB2aAH$(#&y)Z;SRJYenWWn``L@kMY*ZanHr!FZvb)e zMDgk%d3HFTuTtmfr&ra$8+X&H^EV+@YyPp0xe20;k>^WtVv(fu@7B zlfwbHiaQP$^Z{=NvZEjkr92m1X5ImlxsuM|zs$l5-_fDPRteTUf4Z%^LsL?(CJH7F zsSz=&KlFPV=zg9(v1gv1ZZ3Q4uaV2%Gog?rFyYZ!6T7B2@ak>eXc{9|KC|ZgS4Ef_ zFYrkU`Xt|jGIS=V+>bJdE!ZX9T?@ePI3rVU0+D`#IHnX{2y>ZI%B)<~M1+3=lV@4x zx6S9~FosF{D`k9*pajYu24~YkIAX=v0fc6Q#k3R!(+7$#WE_)GS58zW7UPb+Q zohf0ThABIYKH{pepdG65SpY|{ELrtzJ~Vfl{z;RJKz;&fEGURqEPR-zE0n&;mdl#; zVxeNPL}wY9jq(2B6)LjsNLJHJRA+{Y(Sn}*b)8=Qn_tHLPU zWXZN12J)$bmX1Tlp@cVn+htYnV!}+g`htc+QL7V%7|oC(8{4(Sc*N+E&0c&cfDaq_ zR;N^FSMYtm;X41qGk)&AUFxTq#M6UH>&dhuNhXx)<%xw2>faVKd_0Cn0bOJnCibzN z8q>}C*r?3D3%&O7q)WrdAA3Hj!mmK2%uvWJ`;&xAY=)7M9IJSW*e93wmj9jnR#`QV zx8iGQ67%_Pp54KqcT{(?{mOM!3EvLPgzx5lZz1R(6dWW^NuNBr~WH} zKmkiZpW}9NJvRKS(9S%^$)Kyv)w=YnXIvv>VNCw(_DRm{;74SUbp9J3;6N7^wL{Wa zD*sIzwHsQ<2x$VD|CV{QAf>%0Y0CQrj!Xf?QOxW!p3_B6F}lIA7coIuOnE!J;q{Hp zghA58j{#8fy2&Mc5QK<%AA{n&w?5+N>(g#rWACf%m?vNI414x*z|Hd{J(n2`txRvq zgJ)pMBh8_Chhitf($^6`qp)Z>y^jV~sU)xcc-xNCbDLWjmXmwj9-oL*x=Z>!IQwrr zJWiA2okO;b8>Q<$XNKO1zW7cueB^f_r}nTX=Ve!x*+yTV5Ddg(yfo@G;P@Ii{pRB` z;hNUjl@7L8Wy3r7*#5SR{=Se?dqR(J-V8>!z@P&t?(p7Q(_8)%Pq^&YjNA1I+BNr^ zmY3-SJ7@3jIeIaL@3OwzU3uMepUwP|=zn@!t3u0p->14~=EcfSx{=^TJBCffY>RgN zF$C(3g$zY`?5Tn5$p?l$V}KIPdh?Sp?ezgD6K%h2dcFQkPuBooXfURNLe(MAf;CDu zZBtHik|**+Jn}A>(QoObee?oJk%IuNJAh{bP9oY)HYdIf1!nx$#4iMhUJ4L)^7+B( z{fEx|UXA%Zhxk7R`lY;A)omcwCMPny2Kdqgb+iI(`2rMx0o+*u_%MH5XaGLgUsua7 zw~yIDJ1|DehbJb`H4$WeO=v6NOtl*56$FCr5c&rB!g`?M@XzG(0{&#RK{C-nT;O28 z{-8tw;`n~T6f7T9hCo1;H|Qc*d@NYGJE#N_A_fg%hlMaef+o>IMOH)3eZ0sSLIa&a zbwPwRi2ys0AL&J?&{$|jZphoMAVpS^vQdmO0AaKMph@7f_qCVG7}~c{H286tz_W1a zhCtD4V%>^xmt%tUpfDP6m>?vK1?nD*AWs1zQCA&^f#Eo-;cW$BHvA+=>IAo$1VN*C zL=67f*MMt*kGJNZHL?ii=n2}ru~35vF$F(iQ#ikY5I7&>o`HZca{y7J@?H?pfi~e( zAi9|ZT3`hpRT2@MpcI1(5tELr1{klY0xvt?iwbVBH?fn(9=(Oz@grXo5-Vr+_Uvc9m5#YmV1)&|mFVkxUB5WTi=ma#C^4fYHIhY$h1 zPS4|bfbsOOc+#;r9hWbk`$^P6p&#cwO>aE)Pre95C*W#FhhRFOLL9pT?nwA zT{QKcId}^aDhb;8B)K>#5XlE&z(?Kgo_o8wc#v1_^eIL3MJcb5Rm*EtiZ^n|#Dbn9*hPwx)+l-|?JxSf&VbxAD zUUz|dp+KAE!YPn5#@~l)0#i4y2$1GYsnu+7fHt1642~cYFn^|hpG74fAjBvW9c)9& zZlemd2YI7B>d#bI&9n~y(g3pH+*w%mS)e^@?aD0bd8kg-mrSEL0fw9@9U!eLAjB?{ zttm%jK1bLRNL8HbH;|5Jug=eoN!SDs`yCNAfRUT2lk#%aA%D(;9E!rHikqPdh#Ac@ zn9rMMz}^oAX@*3RF`(3Lp+bv5=0+H%O&`gDStO%Y7V1`VvZ;g&7%TA^%BT6nF2J}G zWK<{;8WahEi1G`e|BRDP0*7E=L(%_0M19dMLQtO)gaA?VMDZbjh~f-lWJ;e~2ss2J z-?A7LS}a|h?9Gny)e^uACsptR2ka z1z0g#mcc8`QS4FTD!+>IlvJ|IOUe@U+y;@ZVxTcl1t2-1XIa2UOYvZs%A{Xgl7Os8 zI@BR3pSrSugQ(ApP~jC>arP9ax&W1^N^;V2R3NIIHwv?2HXLB3Q>RYCK-J2vfDXm1 zgXqA$wd&vuRIx^k$5N(3LluW{z=SM)T}|!KI*QH;Bu;K*xc3`cyBx+f@g21~-5fK;9aK4yVUJtT=-?s2IJa=u;%D z7^q|bj}l;btK8UM*-@Ce&FK*MhpsA8HG)@cY4h%`VMdw-FmzUl8S3;L?%3a%eb{r!G>H%%J9UYN{a+s{Rmot znN@-5*8cOFFmw-yeQ%0EO?tSif`3m!a8E@PW<#<_MJ8fNq2~hRggH&aJTGDN_vVZu zotl-fQfQ|~e&>LBr!K5B7`e+ryAvJ4?{3#6klH2G(j~IkC3fE>LESAW(k-pmE$iMb zpV}?j0`l4H^hD9|l~lRYWq*CpeQD979rm&s)Mblo@gt?j$odOJq_fx6czOg4M9XZ6%5D78NX-qIho z*dKA-sSIRwY1tsX*f}j0E{<;s*P1wvbUX!@@oQ)%i4)B^=UMF6OAR~Rlnπ25^^>WUc?`Sgx7BeG$Gi-S?L)|mu zwZ2r7do&f?t!fdm{w@^np4gY};Syu8vM*~*~ z!*%st61 z4+*Uwk(zh{KO7R-|04VJgV6C177?>8TDk2xwhel<;RR>h5drUbKz2AzwhipJNwaov z+t%cfcl8-|Nx-{8upRf+9d_uhUe+!#Y*%n~*B*I~jbZOCc#jCOCji~E%i06M_MRSW zy=2(`0N!VX>@!gBG}`V;rLTvf?DJ>s+rsu)R`<16_Xr^ee9!}%tb@m}17^Gf(nHXs zGU2o>$p|fQ)PQJwgmgi9GU$+XRhei#f^_o<;gV-(L;BC~@k54_LyeQ4fapV>%ER~I zBYMcuYv>U{*3om=k=g1I9rCd{!|@~V@iWM=DfHOI@z7)Km>c=TgyDoH>sSqTj1M_^ z3OzB-Isr%@grb~2VK{vUKBa=4{3-&0mI>M6bflc+z$fDQ!Gvd3oClq+PTaH4gnG_~ zexA9ZoU=bWGg3RFus>UMK81b#Dez?Jdb=V!DRZ7(j^ z4KBK#T+Aw8Kt5cs5S$rhOuwV+F%e&XD^9F$m|*+#$nN{4`7%f+l29H_qA0GfmBH%e zMF>42@g-33k7Nz}Kp4DC5-P42?!_8uNErQ{B-Rl4`FpW)`BgN@mFxXQW;rP3X^Tz9 z*$28?ckxRL!%Hi#%dbaQB_D3H=|HsK0SPLEN$n&B;=nYowA}Jbx2Ja>;kOPSE=4nL zgR1VB#_n`a?$Tf15E^1E&!_w5%wz`Q+c~_Vb{!)+3|mLG#M4xHUQw{n4B2)AF81+Q1PFmy6PG z?P7ffnyzp5mVyn)=ro134mVRg`zQ@WRZfp$)4wMhc&c4$E-lwnWO%9GR#Y9eXPo9( zilgYCK82;Clv+w)e0a{yj>aTy0mgm*F^V0F$K67*fc;#R8BfmIqN`Ad_t|4W3ZcDB zgreUGGjW=_eQ&g`$({@~S4?GJq^0$qEWPmfGd9{#za6>9{Dd)Vj43TU@;NbIVi;J; zOLqoxvnQW3vWHfkzVxeNK4W+ietN0|+j)1!#Ivji`j$QY<&2I0h*(jDl4^Y-mphp9 z^>+cd$?y}nFg^`vG!vt>T2%%;JGM}o$&?r*$28^eQk;KjXFTKig@Z-_3x2V>40uyP zL;bC2zec9&3yrB%ZO4l@3c@`X47&DJapG7$zqb$`J8AJc%EV_Ge7JSgQGq-tOpB+~ z?&_Goh>K^nys2?wv98v1SV(P}I(wVgNphv~(f;|BvV);Ie^(ZLPwi65k2oHKgf0Cm zWp|tLt0C=Q$g6aoZ%?lb15~EuDWU2Lo8NH&Bh`>_!d`7gMycZ(v}V#@N5vjERabNb zBAXIujX{eNnZEpA)we>mulvk=xJO;yr@LL=ywCI{bv4fp<-0Y{L4Vh1ntVlgBTk!) zK{N&rR}Y%pi^}6u&zw@{!(t~!JMGvHQm3Wbv~GS3v8hw&sQg*6)V6O?2En14R}?14 z@^8Lrdz%U3p%h?mR})x8Yul#POiO)JxS$`6Avbc=v15DT6Qnr!p?C@>WsoD>>vlmf zf_BA%d+3uQb;aaS_4|JR?ri8fv?ZBl?z0MHr;(8~SsEX|fjfx6{_T5iXsmJIKva;e z{e|oRongQ8iR{ZC^rocqO5t9qPy5}z5-_GSSmR~4i7@(Wy~`@Z-+Rco!dH7H3W^dn zY;B4mIJ)pnEbr%WlvYuQX{M15=@9VI*zp)`%E(WzZac=$^+vbIkm{Y`W}J#-Dk-9e zgIN$|b;ew7lp0l3iY*Zs2{SV~+AB<8l>{`^#}6Kyj)6xknWoY0cQlvPU?&iDCCDO9 znYbF%vLWfy@mRD<@?D85NEF1F8$|fdh`1~?*&ls#~@U0g?ebL>($gQ(47nh(+@em8G8!Tpe++em>k0l}l&w)%< zd4U$hD^+O(r%Q@Wr@U481i|`ve{T6 zl)1?>8(T?LbFb{)mGT$dn+bfIV;|aCPu(|pFT8Z4>@3OSaFY98^rwr8`_|bTcY2dc zjOU6T80Q=d#O7dpS5*r_;w)T`FiDDA)n)ab?TNzf$*k>7czJ+xNg;Q z#iVKeZVE?79M8J_$KzT0$N&J7Vrt?s1WZZHW4YL0nbQ|ZFBL-l`|<4WTVQqi-gpd^ zj=ybz?deqF01op+g;2!l>0bquDAEkeRJ2(3x(N5&zi4gkr%Z(h%WRPvn1fOce5DKZ zzaP(%y%X@c6*D8zezVfr^c$^B$=P(d(PEvQZMXrk1=fwLLFYVMSMzftk}RSiL2Q9D zrHI-@DxSl#>0jY$gny~onW-?wbP`SAgN*=fv>e!)e>_QD2Zkgg|_)g^&_x|X*?9|=M*EF z0$(nM6s1$+M&ihen@DkDbVp{eF@DXQ(zOvwR3eIL??Ng7fc;qU)LmgKW4)GbR3Bxj zikJ+?RF+XT7J{{tgG5R`sS47;Rx`9W`$A%AMXG_Os4dCP?)2!sD^JohpV(3=fV!qg zBJCHvOIJQA!m`BK;cHE?nHct@3b_1N^_F(!ue1GhDxhVQQ0tF9YX=3=5_U=ZI(F5Z z;9p%uU~>Hzs)>SP9EYDNH}dx7ofL5=8xEec`byx|APOjqt#>Ld=ZFG|w8~mDqJTm- zdF0=WVO=4QN>_T^eo5ML+*LQy;N1?;8h+A4`Wvn7uL8<*Jr*;8?S4?-j@=;V9hu4y zx73c!Ft1j_$3JLoB3B20(b~vt&i_Gcb>SAOCCOqM$lL7^TYqg+*?J(!EWigxJwo8?ry=|-JReNoM6F& zyB6+FaCZp=3+^Fk2=0*J4z(-)zgG9!-Rtbpdz^D|F1h{2pla5f@B2JvJ5bqA-Wps} zG2TJib!~zH%aRF?$QBdQj|UxoAN+oO{u4lXjKP2eP?(*FV!qM`R$NZ<{eJ$tJo{e* zC|6M^Hx*+W(@pVa_p}f28{2ms_wKBEB#gIq>p}8fv->>VFzQF%Q6^ z{1ZSiFXX2jV=FN`` zcYY=$*%@toGY>JF;GpAPjgE*uUL*#gl~N!o{D;{DKZ{(5>oN%xXEj`W2vk(OOwODv zT_UMecxeft$`u|Z6FMq-r%aPt`a|PXc^)|=jw=l{Bvi!&SqV;MCzT}2k`+HpG4$Fk zqYtf&BT`|~f(UZ9@|a;y=`6M8(lY7tFXO3(*uai=q$qsANt zF@>u~h)x%xS%paw{V|&?6oETzW&9B4VDilZ(dR1U*_l8|7O9-|&1yV+a~gMJ!IX6I zjI!2>-?CSE%19I{0}RLea$;IaMbWF&?F8xav2ZBCLRAVLRoY0jxX>44L5@gKa#%=V zFv$H?(~EuI>wIsj9Y9F>Bda>WVLFuheK2|ps|%@bud;NVPW;rx^}1e5>J;|$ALlaw z!uL1zd;Z0Uip50VICQj+n9EJaykyX$#NU7?9AJrNK_lZRlvDtD&8sTFbcihmva6Kz z^Clua?{u?Qro09haq^-FQVR^QR$S2tAO>EEZX;tff)BN*4c(=}$C?X5$GE_Jhcg6q z9y5w;bqR+n6H4KH9_m=nh5RN}ivYHt)yc0Ysa^LRPU7N+Y+QF{AkQ{=s))QVbuzj% zy0ihV;SVt@CD2a{xNi7pgpb|K0FRIchK!vkuZ$ZO@Ng1pR~mB`+n{L3AzeQTS)yvV z98UTJ@&Iu9FH<;%-~Av3Mq`wm@-|k^{Sbp{V~pP9HeNTx6rRu+XQRABw01wt?+gfc zpWGo6a~l!GXaYql@8XlUjV=qgeN2tpJ?-G7lY@6lN#kZl11rIH&8hn!!|)?0bPmk; zMz=HIe@d!#Q${CCTSFPJf^E$LQ`zBpgJd)Ly`FVF6$_49=Ge^;Kx?C3*OJr5y%-b7OC7{H%Cj}Us^ z0wn&UJo~&&Fw-&2H+_-d_q;=a**PkwdYPQ_yvy*9@@#td^BzZH=cJA5RW<~>#y`_J z?e(WT`+Ojd*)AQ2BZp3veD`U1E3KQnr=Rcan{KZ z*N6|*0kS|Ze!m+q8m|!S;$FZ z_qYit+#MqFSsPpW0+%aQ5pByPbQ#}IYYFA3kIqF(pqD+GlS7hCj7?*$^z!?}=Pg32 zjc6AW%Bpt2ZW1*}R88F@;ULqsEETju54fpY^84BU%u?2N3|EjZe<@-66nGx=n>e_) zRufTvLyF}NwZC_%edgttANXk#^NaF5*n_G3YrylN+T6ppnb+HH@XI-UR|G@Y=leDA z>pl71qe3q@)rJ)eMc@);;|@y@gy0>7loCn*Iba= z|2MfHp8u;{kk7MLY+LQ00c#r#MRQZR%kAoTeWIOyEhG?e`vXBo{nlWVz|S51OtYPc zP(;-j0vs3#bc8+J-R;?5;uByFdboI?>Grn@Ext28&bKN{I6Ia3Dnb_)=}E}@vPec; zUyZ^QY^>Squ-3HFVnI)llygk^O;;!6qGw zaA`19AmQlar5=tJ!9eB2<87-R4KH!HEHw$AZloS~F}o%17(5D!r{1ry^dj9l1e)N^ zTW=>wSF5AUlCdl(C@>+tSh_(n)v5FP5=TO&)Y%(kjQl!C=(qM- zQc%ixv^HCGxV38AgVvo|BeTQBmnT_9iSt|bo1)dZ#J&tpo!qf8&-+0d`k@L*YD>9U z?^xW4Rk2eT<)lgEx!k!44)nK|)7(=HfI@(a;90~2(!1Gglri4fVZ?C9E18%3(Pf#$ zn{M@CVH*)(IRce2XcAfNIk7+tqm{#D_G7e#hmxUD z8i+FtU!Ytgt zoUycxR_;77FRGb7t&Un(7>^J?RgTlHFwu5Lz8MQTLu2nP+5R+>n^ z94XF1AWv&hW*OrrFUsb_Ay8};Z|t^7D8toh!p3NPnPrWW;~~87v6 zPj+;tMqXh^0fxv`?#JWk!XKUGjS=1on;|6sD5Z726tVK?V?0WGP6} zu24CfB#p`5_c9L0RliC+yezS-#kCUPnyHn;B^_rlc;x62!%%#A%_xhjt(llD1z9xl zDD7!!7cix4?u}>v%3;!pFa9p{;X}35gMHo*if;NnsK!;&!Qg|Y$oW@*h<_Bb*R!~u zSlCPq4TVI{Lf2w0ODcsI!5mWds$Me>wzl9x5R%&kra5RbVkOGas+D@hu`sdTnW_jM zH8zbJZ>LdzAoO=tmJU;}1%*R{2{~&2>S9wHdtH33W74=mJ{S*%DKk8l(7rljZT?oV zd$0Db|AZt9yk3V3rL&gXQ~D0o66-i^Oo$;AF3VFPnaN@=M^@uvF-KJ zAbi$s(t7>zSb?3YrLK+-zx}$2O*f}AdcWlT`ZzK0vn2P9KO)$W}~)KBHEjWLyu|PQijV6(3tvlo!b-vZvh<>420ZzYwd4j%7+3)uxp6)2=3I z13Ia6u7FsFbP9(odF|;uoqV*t^bYKw#LUiR`r=%*Pao#GyHI32rG~mZNI*Rl0OK zg;JrsT(u>d1uXLx$>N&h7+YUT^fLS^HB0;KNutrppQF{dSPq(gzEkJGi;9m^mS%qN z-&ChZ8=~VaEKfdtM+3G=@lTzaG7GXy?8~=R_oNxl>@Etg@V2*ZJ7op={m^9cQl2(W z`bI@e-K{ELKj(z|VK8i6HjA&P+TwS{W)9t28E?y7PijE-?gqY(m-M!Z3!D(~n@Bvb zLFVW4XvFDF*3J)9*@fW=?M2A7;tEu<&KGTFxwIAF#SmbNc&v1b#jcaO4>40Q_onrG*r7Rh%uXAUKck9vus$aDI{*tLit7*86%Ya#sgiqSUskVE5f{$bst zfS?Fhqo8ZKzb>57%$N!#1F0Ob_a?+Ek6!kEWEm*=fwge!6sRCHzowd&QolQ5iti`Y zmk`0b`9foO;J2}nkDU-HvWp4b4aC3%_At|2#1zDA=xFYAD2Y&^CkX7Npl?mmXI$_p z41Et5O)69)yDTf8+Sdv(XI0Llnwo{-3{gD{%HNU1Z7dlVk@=IOl zCG7qp&-@F+FNGoSLB{S?0k6jQP;Y}m&PE>0JURW8EKl+zJ=d$y5qNV*4Tqy*CiL*d zH2I`tgZ}u7wMN(P*U+d;yluWfX7j+icI*c^teK@iCgYY)LLf{%y{hNpBDz7N#|h8bGn42cKex&{kI0omU9zyi%7E;5#_ zumyLV&ZS_iIkaHCa6D)fBYA+`XgEZCNq%8zC67iY4`|5nQ+&jFZ7=y#d__5;;g;hB{Lo#Cc`Xj#`0>5YSY6Qo3E{Cl1n@tel z&MyOx^b?pb;)#M?fcMytOEunBLc%WYZYH)C1wtd0ww7mtQc|K40rs_oh=XRJ>2(R3;eLm`0-6m6NxIq!QaXqBA&~fh3XpIg^pp3!lbNFF z2#`4yni7h2l30%b?9uFRz>M^KJ~G;ziJXlIUOygn z{Dm{kSSrp|YV{7ysICpERa*Y2we}I#-i_gJa}AVqT!+ke4C;Wmb4&+2fW;A+m5;fh zYq}vFpojwy2g(rG)^NQ|W4s{o6`=C(v~gdtAZL{94MK%W%7|OZv|KR>h{j^rgpo~7 z2N7n`nP*X`XQeY{Wcpy`831rNvP)OUXoHN2gW&~lF&$>J>j;5$ggH%wIXFp%NJZJT zgxJklIh0{A=~>VMSTJCY%rHiv)t5|p;m%`d%+;Bf z^q$T8xspeVr?(9j$XCcp4>iy@%*r2(&c~$$Ja6TLH*?guq(>A0p4K)l-vM{gfTxF0 z*G`^OMnI4o44HZ%HU~8PF*LZakVPD@ssNMO&$AlMzk-#mG%tx;F4s5&os45b9Y%G%GO#&VJ(bC+Mvp*ou>A-khq7V=yG$g+cru{j0J6yIlCD1kT0 z^`HAS3WQ6lbDK0Qo)P_<)=;^`eE3-(4mCz9`;wo5Dk?NS=H9-B8hEj@v zEf>s7AX!r|y6!hr6??rcstOcNfv7GI&TzmHjySoiK2>8icvfCOfB*nhS`vfx*$Vgp zRtcl1?6ADdKZ~-tv<9)!Kq4D>z4Zo;vX-<+87@f$JE|7?d%lAqaKIjT@|e|g4E&8k z4vfw<;i|tLr@i&9H%zX7jG=vAt=GM*hf<=25o*xXY(P*_LS7R?>TZ}A2W%cUbhrz+ zSRvv0H4@}B5_LC{tTmE7HBt~aQ3*BC7&X!PH8JEgVs$sMtTnMcHE|F(a|t!`7&Y^P z{hIl6ngzR?h1Z&Yd}|OT))b#`d?je{P0p9kX;JKMQC@3NeQHrBZq*cO)i!F?^=p+k z0$fzp?~ilKYfv=WE$3UO44Ef2Dll&t(6VG}sL8XjlvRs6aG+%!PG~NRpkt^3AotEQ zPMTjkorY9h5lNDF5;cqRzyxNxEU7L?j4t^Jvy~eVw9PL?j0*#7e4ojW0^OyE)t27H z-iFN_=8)9#gxN?;C%&f!-0Fg?d6l+5PhSJ%o4Z`7dQd{TEVgSopljjhINQMhHs1-_ zUwcN{$hYTj)W_BKqc&p(BNQf^%wC;S}8|*W?^F7?^)Cw z`Y7|zO2D?uHT)sq*Eg<1DJ{9&OjNq(LuAi%A3@aELf(kP#OF8NWE*@^t7^Msu zl_?r^9T;ui8byN{n`BMQQ|#&=bmegpt;)mA%8uQW*zJ>aoyrnlK4bUi(hUkre67JAiK81slbFcGp7N)g zc^03G!~W(-_dQp9$sc>=E!|q2__`zZhBDojviQzh>^)(+17YzaW$csZ_S)Q$dSU0^ z!q^uibZ2PdcP!WsXmn4{VlO1v;AdLsGcnls-kwtj%UzBhPi@1jtzekL+2a8&yi!^VPeBF+|7K<^DmbO-m9sH8P%4+u}9*Z-c zmYY;e;3JmMM_LhdF)=4B2`5^q7f~5wEIDIZg)>nl6)Y7MTD5gi4G}CY5n7!*Q9V{H z16Eq2cu`|gEK^cib5c=DV|>1NTe}yaH9D;gx~S6`@cj#o>$-^hI`G38jn|Zj&lJ#4 zMTMDk%;Ci*@S{q2yiMdwN%Yyf*gRnH6wOB`k;Hgl5@}Cp-q_8gQ`$#hMlDTBo^YNM zupp17=uD_2zAmR{tjKX7Yke#}?_H&FXXA@?GpTg7%0w-?X!*KOJ1g*0JWWr$(3g0{ z)>`YYPVFPC)?-st6M0rs>x45ZR&(os{*N?+ABDb80hXL-R>13mYcFcwdMvk`6n020 z_f+H$K3W_Zlb=kPpRs=KJe%lV7ulE+yy$8Ftz!OQ3|Kd&xvLd;Q2~HeXrS5nVSl!L zep#*hyn&d$=`6H?X1wvkeKQz#3%-{c?-wt@uT7Z6Ei8YpS%>mRvU!3$F#N$ z-M3Nmw@XjAp7yqHr#72Nw%Ka8ORRU=Saw{u7{$qUB}I27rfGRhwuGlA_!GVf{uE-# ze

5t$;BpDr&0HyRGFssq+(GCgAhyx7Jy+HX;lO!hD?aEF86I9BVRcg3WQ9Bp6r? zXjcb%67>U5lLJoI1K-{Q|DOi|uLnV7hyJgaB(wX@uiHXkE_oZQVV2remuEUO_!VIO zU@Q%-n}iL8Y#?No%H1i+Q2bV)O?L#53)oj?pEzc zfFgd;A)!wiSN!@)^hFRBt9iGgzGmBx>W%#uKzRdY^T`)~bo$Bs zU!N}lI=(dfL|SN_zp*a}Jd(~vq0daVPEC`(bYm6K2A`i5orlwoq6OmOZ0O-*k`sQ> zB?gm`n(C6*i_pd$_6~_ri(Oz=i!siSQi2mO7cSSPFD@xu)mgsm`I{3iqKdc2+VZ?_ z(Y?45e~%|Ab}butEzgdBkbkYbajgozmVIIr!Mt%TV$u%0fq3H#zudgVoZATbZTv;c z+*I2{^w#o&tk(Nmb52G3jaz5%tqb{`+YBz3*xlPCiXG=W-}fN6FL#zyvUb1kjDYtc zzj27Q@1ww3G6nZ>U&O*T?zJwMOSm3VO&`(&A2JIbvcEj!Zan0J9}3AIi^U#GO&`kw zA1ewTtG+zeY&_P19~;P@noJ)$VEBp>gFwsC1(tkh<^1~=NE0o{xy0{aM&SHUh{0_& zR~!`5KJ(wzN3TmEUv`M`%lm3uq`IBLH2CpVAP*_cYP}Z%F)*q5Fh7Wm^dSF1pWjW2 z0mQDGKN{Ki=Z2g^;t#s+pM0Z&n#-b<_veN@v54euaTYA>yf%H9BI_K|Wdi{)lenH2 zRT+gSoBUriiBrVbzVyC4-T(83>_rQ;83cEH&=-t>L}M0=#_LV4g~VD>7>c8Nm=}gf z8&o7mYzVm_JING+!btaZsG=B#5h+#ZFg92yA`=kKL)>H!(L5=2ZZ#nMWGJf9?Vn0#ha41)>xGQXlfr~$!mqi0j| zj9a#Hz-B4$B>i(gl_gs^r`pK+B_EYLGCL~(L=o*kpTxsrTIjW);P#8f)t?y|W2kh) zX7cbq+|Vj86G_sfy^{@NlrAB&nklVF(5aNGx+w_`bMpjiA1joX+E;(9cJ`=}iiCbp zhv{*Cw2KTNQ`K9%!#b~|ww7p5-HnAPH^@vuY z$9_$t9>JRKR~ziqhi`S8i_F=?xH?ZqB?sYq)P1-uDA? zGOESt{AqLs$Uhu87Qs>a5q$h>R-~_gboL@3Hy~KJyB;fqlN))TmpgiTH?J=RFq7 zr&`#6OZ)47_xVNsVedAJ1oLkI^VdqW$HUU{%(mt7(53d>Pwl@RPa33#a84Sr^B*0! zzx~?TocF}&KP!edefkH)Ad=Pp^5>jH#O^J%7f~#<=5G2a20PRui(N9s~Fs_6s z#7M;?bG8xpl$2;{pwjQN=eis;v&diy?4dxMByl*JpTNKhd{nNB=-HJt<4LQk0 z;d3!w`(Y`I+eNI&1=Y|HlI%99G0wJhobVxin78ZVAI2}a+(yUv50fQicaR%03`SxA zTIs4f1*9;n>FO_E+IdbfdPvZe#D_{rL!SjuOAABTg<0p}?Hv2<;zpFplFcCrF@E_X zg36RXDWGT|g*9BJHN<^lu~C>XT>ablWml<)lZKMk16LBXk2q3ny;s9+80;Qu}eaDpGTd zkmVcKh*xbXrf6jl>0Fsh_Pi#?L1&Z?jn>%Dy8c80AFilVHJ(m}S;oJ9DE3bATQE*b z5-+WZYGl=1-X%||XnYxG^!P#)S!=nXjSLXT zbj=!D+G-tGLhbpYn_Byi$Jkm+y0bkub*>`z^7||NRuPwVK~v0XVgq{n-&@^-RBFs@ zR+9#ZJd>m2Pc6Pw0Y^mfl9SSE^y;&L6DGQ#tf^BgJ;RYfO}3UIQYT%Rrq!h^Pqzw{ zvv+dyhD+cO-nJ&=6KP?k!6jad_6kx~d3#i$4JM3^#ym%9ZwJ6O9v^5_rOuge`^Uj8 zXIF=DjbuQT%nk-d_saKb$xJ^LF)A+#`n^N3VI~uP-=+6`#_X!_n%cKqya~V8jXblM zOdmXNdQU&Ic^?>=)(zu+fg$5iesVCKN61X940n+Jrf9&o-|md!Tonj5T4y)ObO!3O z7>cc$vjr?W5t1E8RIbLPrY}*gfk|3IM0Ss15%;Y=`|G(i2WKZGj#J4xq<1)ESuyawJ{s;S{# z;I0ICQcv_Uc`bII9TBfgoK9Y36|@V_zA|Y;)|B_#ah`W!haA)IO8_kaJ+m)Qm56fp(j7uSml^%-muaJNlbp_rHED)%z>cL3ZxbQJ^HomfdzfOJlvvZ@2 z>?#h?_>&d8ugY#6U)PdIul2j`S*0qkO7P9+U~-`?R57_PV3AfYOuNO_*IQx1*Nd2V zVR?GY-SOX6D4E^D`&(5HF>d)6pVpboloD^g`k5paM|xUZtzTr7^fEl2`f*S*--9Ba zZv5k3%Awbu7Ef@#Yv|BjsP)c8|LjeKbF?PeP_)?a!+DUB5IiVW8v(jMY{L48f{nOC zLI8lUG$x6_lc8+QG-}#|0?WJy_{;HYr9h9oyJvrRi#fTgp%=q&V5D^5al7JOhHxHY z5Q|IzTt5=LJ{Bgw@Iyu*!yA=kcNk7cDNCI!ZdN@j1xQZ;L&hIK&?p@iifGIiBG4~L z#SwU#1P#X-@{u1HB3%#hyhaHETq4*g1yU~LY0ksJvry7uF!TVLD0z7hWe74@0w$aW*W4Bm zR)L?gjPEpn7+}VhV+;J~4amw2YmtiT*@2e5#!NuMma>v}lm~!AV!-Ors6!-d+LUzH z(Xhd>Fq|A-E{Gh|fPxOfP8&dLrb-QLG`J6Ku1S2$1+`NgpmvVw#H9$=4`3-`LpaBb z2SI@-LmRh=$}<52Hu(OHoO!NH#O|nR<_cgJyv=*eiz`g~z69`j4E&Zf~hfH_Or z4*48Lof2lWgB^|&IbIwvZTY6@J|WNzrC$<|$0>R{n+Wk5ze7rbFgX{o6E{j#*|hk+ z9!8SSxuX$E`NE2z1CtTL3C1}RsO(5h!lcN8!#9eP*%egZfF)A+Rs`8PQ?$ROh#K&S z6Q+7fr}prq%KC80XQfg$rz#7`su?J%5~evyr%hi=T@&!`4@lf_rs*PMk$EVVVyBxP zsaO+|SyhIA+DX&%Nq@JO9x#;d!YJlWnDGJmqZgsSPp6z)XGZuAbAW-2^0$oHE$%P_ z8!G%v0@uu)ex8hq%q_SqWO}A##>}*6vy2C`tmsVsl9ZfI?CNiv1e7U|p9q+6)TIJ+ zU0R|jMXbuf$qyGvvOq=zPMK%dY@Dqu@=V~%EN@p9pv8`zNIRRpp97{JP=ib>bDdKV zokBGyLKQ|{nUzCrE>g%(XBSNtGz6)1igFGR83;&Dfme7Isdy)&nU>v|?|AYU-12cG zlHA8Kx3=;T%oPSZ#c`+#9E0VykU1xPs70l@cRGbQB??}P3Q+kA&B6*%d<)T!%`xpQ zfL$_Zf<+II6Z<0tq2hbu$M>YiMdXGilqh!8tM;_^#q_J<1moC=S;a1m#X@t%Y{z&U zf+YtUCA@}YEhW$l95B?g0;B-GL(0;%av-P(f%hRxQoWSe6?75|J!esM-fmEqZVa?&6bow%C~o8#YP8yJM2BmtVr;~& zuG>>=3bbpod31wpITNUxwo01RshTIjQq5cz&10d>oW;$fgUuY<%_Fcaj8rYdQY{P? zEnh=h-dr~+KT#?w39C);vk0}~VYLogwhkGzGBDQ}<`k%|wJtukVuZA|O0}U|v^9pd zp%u3^47LSfx3NvMx;r$v#7esnH~UPu`*rg>KM8sXQ3Rd{2BT7ht_g-ekw+Q{MxPXV zRJSKiv?rakf2`(@B`&i#sfmATdTs2?ogmMD>U;!t6~~g78g<>ec2zKwS5+6KRP!5C zHP&`_{pRd$StD!n>pn^9?#dziyw-hi(cLFRh6~j^x)W6j(XcmkM;&7g_~EEO3NW7Y zt+`j4#gHz=62J-+S`0y>_d{=jiF>z&dUuU__x*ae*6`+pK5w`59CG%2p!rmuNqFf; zelyXF;`-%QsK$?=?Cmls*Q&VBDCd1N??Ws|{qhS_Ql22N?}HZHtR42_6G0w#e+n{I zYemY7CGf?L67M;gXI_)g4gla9uu3jPYV5b+7m_v4!R`lKxO8#@CEbb{ar^t2mPJJd z(qMxJ$S-5gkrTgX0(c(!(c$42k%7~L`8L~tBLxlxG@zi}5H-I9Pa(w2K`H#U@P6&< ze;B{49OJCtireH4S1=EfMGX#IaFF&7hwKi2ryThvJ+fdv!fwc?Tr&&-Fi&j_1fg+y z3y((lk4CYO2!&LvjcZ-kfRB*rH-?dPW z7atCzR?1giVf_)9f&8v+RK2l(D(#QzDce>pv|Nf!MFr@3^k>tcd+6JCw$|Z(@^k91 zwEF*1^<-n@_n&U@&kt9J->Me2dQzCx>bs+p4RRbw|Eh-z`pqEf7d_BN zLwI=zF7cgJp@a(Kp>_}^ukBUFJ=IEw2%AQICTX-mqM)~q;;SmQo1iWFs+JQ{zP zAqx$=h`%@#qqhPSZDVXjt;jOYV5P@{6_##;N$574ND+Re4rD$aDwWm3u-h}%`pG>j z%Hb2Sn*hV;1F)6(=(Cqq5C4FffuM8rjfSU(LUp5equbYPArb zVtZ29o-ci2)%-6)RDJ+g`nccsEPgjq<-hJbdwtqoihS>JyZnBp=lw!#eedli`^)pg zN9%z3lanP=-^Z+|gjoV()!eXBkYZjL0 z^^000HT)tj{mKG)Zgd!AS zbxqN=9(&A51QWlelG9tIH&_#+aus8oGd@pRzYh|SHpfWG6y>3N56V)uK90x|@!01s zik7n5kI4SsvhUO&Am!wolEZpk<^;K~x{gQWvhW-T%-KnP7>>w0^Eg1=KQa zjVK|#IW!Zm5=}}-E#2a-G*gHCI)RD!wB&xIWNt5>wH;pernS=8rAs(3CZ-(s+RiwL zNV{k{yrP+#$r!{W^ocC3lGwxEsE|mfQZ~6NubIiH;aH&7B&|B>s*0qaNwP5}yvF(s zv;HiRZfjjy?V$UK-X@V=r)PK_L(7Tr1rvYIueAE0D`q{Y6UlyiT7P7UN^{H!VeBH0 zMm@VzhPd34=J_bf!$5P%PD;_<70e7A*x<94K*E zYp^XvV|}=jHpc|KsBIAD1w^sMP_?!mqKkgd4^RHiVRf7CQ=?2FG`@3D+!oeSr+lLc z43rCuH4b!p=hr$704mgZgx+UqN=GwD-tmSvvFGT+`=6V{rVq!UUdcvw^oup}m(}<$ zFE{KzpiZ%&N8s9C;~WfVc5HoO2QfeIkC4^!EH%To@au zl{m!3c@d?7hK);~^p(}q_1Nqgi*P1$SiGqb+cnpc)FysJiRSIafn3pMmU&4gC3!aFx zoyO1a+Gj(lgwL#TpSOZjI1VzS@bRzCFTpDjwWRk-BIPM!4TI?*c(jNyY0>z7|6Ts> zw?&i*=+lT7ouZcGHw&6(aoIv>M1FBj2KhHAvyw-X3c={ZJ{gGKC1fbPF!$C*L<8x=hnRuqrj|y1!UKzb|^bDHs=difoM`FRtOUAfz4GQ&Ia{nXDi4m*{XqbX7SZQ_tbqp{I*o7K1||1=Je z*wBSIe)=TWv<0t*G%P+o!H>E8bKj4oU)4D$#Q1+Vd=ItJ2%Cu(=CJmR;QnEell8pJ zsphqP@OnS0)_a*T^Smzvewa<{y)IXKIW_`6F3_d3_r^;}1XwD&7Wz7Il?oFUX-h^C4I=@gn`s}~ADv{2GjJD8e*vWt z;aQZ)RSIMML>D97h?6hjTchdbdf07>40G5-Xm?r{@{;uz)IL1joH*sIAmaN z$V`xTO*p+zI44L9+Df?8PrUU`yw6O$ETX%vNPLy|*uGDM5df(|$HDo4kQ8tyGePK$ zF=%!^n1o4K0!cP9NpP<5c+p9Cok=7@YFG&KMFbq#_DZzvj!gthy3 zi15D}ns`2c=}i@)|5a}~`M2KmwN={Ib~8Fugikfu$D{ne=}jTmHH5Qhl4Bk86$;v8qS_!FTn;abAn?~#p#3P` z&&kf_gBj#EeW_?V+=q|?LG|q@UzuVDlHDvv9>7y+SrlK91~miW=sQFd`2S9C3c0Sm zCwCixe};!}dI4zKW_pMi5Jb2(?J_8oP=X*i3=PuI)JD`~P>&$`xj-pRZUWJp;)LI0 z!qH8qQbn^|-sQ-FtmOe=waB{#fn?$LTfuzJ`V=t?sz^Ih%x$*2Hrym(B@#pk2Aj!B zOgMXz$hQ@sFb2B9y;L;Zv0df2WQ_ao6apR+v5+K+^eiP;7ywX2v{01c(ftC}!49H( zdzt>uZRPUjf)$fucR@6kQV-9~I1;u$arVY`wE6Is0eGd!Jkx=T{o zsJ;8?e+M?!{~3VDcE1_+Tl;F$1i$WnJC5&9LsQN~V>?Qzj$|+0<{x@fNCK+2@US4m z*>|zjD{Sqkru~<1Dd88kCzqBf=Weaa_ROcV^Rwxv-|n@LhNjlunnlqNef{%48=Cc| z1M~k45&n9=oALhj;jkQ{H$83tE!x%lg&q8S-9F;>a;Q-!ak{pi_{B%Ndk~BuFMjvE zIOvjD)>l19{8IfFHaY|m&WHa6+3=S--v5pW-_hy^{!bC%;Qt3i_+cKF=AVXUEbdhl* zqU~c-&c0VUn`q_yBMQ@Q*;l!c_NL&{*tBQYxA(6*s+2#+lDy}&#KK1pL|{|FGd{n* ziPk2C(Y#X5h7fraq5~_Wa8|SUap2(5p39~D`^cgcuZwA#=w;7F#IYQ&MbY9aBqRYL ziCuD~V2Vmb9YH{<;n1hTxe5h%7!4erJve^cQI-0XZ&|#h^b8i88ZIco@bVFXl}tzz zpvT0QhZb&n0!8#NCWy~aq6}LA#ggr6VOv@oHu@pbMAe1DvlXEl-%eW(`U=VsuAhR8 z6p4fe8#C19jR})wG8iohr>riGqv6>h{FzCA9|@B-jMdS5^V9OM;B6x)zQ&4PX=U8- zwkZ>$H|3k?>T}Z-$bYG^l~Y=s&%SM`P^q=on^;}6pOwh>sCBea5=8_T-B`Di8d=y? ze_p=4W$AsXb%|7ZD}aD+R`jNLgezJ0Vc{2ZWGIzswi^LbMVjN1vAW0lFl!jjsY9Ig zXn9#5Q{^obqx%7Ze>OA;9H1V#9EU&L zMiy*b`!N#@W?~tjgep`~krtzZ)!v_d-R*y~tL+HDL>gMcguWFM2AtU3Aea4^YbEqM zCVUq-QfyJpe3k1oMS#d4u`pdF?s;N5Ba-32@sl#f9D6ZlmU&?PR^GF7F}X`5$pG2F zOHv-&SvT*AYOA7MeAusrlI)ZG*4I^Y;3cuQQuPl$mfOl9JkIF*51R@tx6{eRbL+=z zi~nqB!qTskfjLK|*sQ`+{FSq`()GvpxZPkN@FrIJ*)bavf|9SfkcOrJ4}Rs}4b6>* zxDeHAgpIeOa_qk0lyx37+@!OJ-xkMnRBx(`UXG=6AG2B$ZtC>XPox8UG+#*mCB<_J zRR!e(^$rr|0Yh-=B=QXqW;zn~-*ZMN08ZBbZxso?WE4N=9}us0c~%{6&HQ{n@Q`{;eYUd(J5Qvs?S)1AxpK5P*5>$)DX?zKY)) z^l2jgtI>D^*mys6tRHPP!6T(wZ?WDks#{?}IgYXOAwr20;*=#q^OP4!=XGtR#*Pwb zjqKrgjtUpzbmBVSxgnL`@9sMZRPP4;C+(dP3J!V)s^Z^17DS$)nCpEG_cfN_Z)}EK zIG)%)uo+<0Fvw#;ye-r^<8P6h%cgSp5vR4Y{>T$Jz2@N{ktdCCKp2L|c+1fY9OS9; zvb7i>ktYMX*6h}5y+k3c08*t}s+kOCN^y#gmYTI#2U2@)rS@Ngot{_6FOgnGyZs6f(`CR27P6e-Nq^ZmM26&hBBIT+wEwEvWntJ?`5;W zfRmNk(U^b86By*k*?N~v;y#j4MUq5Oa6wv~c>lH4X`LeddKD7;3z~5^?H}@lPOEgI z$I<)@>FmE+os6{EZ(sHEt$5*9%5ov{1o}+rx}NoMMPPAtk%Y$G$w}7~FYWQ#l?So)l$k$K+cv ze|%ExRCh66TPiwkSL!|fc+wPz6)huTDP3J%{dwScxqR~7L^+8xKub+zNLh)YDT1l` ztTU?)x=!rnUMmr%JAa9}>pDV#1~_p~J;Q#h49gz3cp2o>181aQg?e+Ik|@GHB<|F@ z>uc}akE6yYhkA(=8i~t01Q_h-`Wj>VRh~}aGJ!RUQa1K=N{KT$Xy`3tzuflt&fq12 z@cD$?FbVezEvnUNH856~drIo@*}Y%ki3O3u1P$GFbV*gYd0wCV%wbH;GF}$dD1^oC zJ2iFu&5B{Y;pIw@KkJP*TL~)nnxc)+-HP)Fvk4@`0ak9Gw_7^q`)T%tlx$-aX8vZw z@!RV4hBNUm((M(l>HC^Q@R`h(uUcL6_X9)c&dpc{49~<7O;(P>=-Aixics#}&fR=` z70=D0?f~AhEF)3BgVyR^_x`Tm&UU|h3lgrj3NlpuFFtS+bsv6-&8xYBlfvY0_(o68 zTRPCJ+ITDa6#e<|Ye2!{auSCqiXJ^mfzK;`@`Dd$<-M(ju zf!3D=ha}ctDjVN><66$Ya_Mw!zd;FFWY&A7ii$!`whK5M)Omp+_>35n7)pkd0wXml zj^!EY#t>!-ZO}R}7Y*{H{rnTnQeHGvN8L-G3GW>2R?$x?%MtGWND0VVs6tb83?|DM2@QBYU0&hgin%sf^sH-+8q9QevGISIUrByGV>5=O$& zr=Gc}FzHgdc$-F5K^IVF92SzC!ni;dmkn%~8B@`aV%N$^nu`M&gOaJ+qz)#r7V zYMOBslgi|L={*h|1Ld=ItW2NJyIeE+6PVj>@4Ir#()6?X98ds(-EZGJgM*u^L^mW;%{)1i_J@`P{7`cdp>^z&UZA9;4=1yGb+wzcxHTDN7H zV@TzGX(V&D($X7bE2Ruvlrf&saw;6F6%1TiQKlWL+WC^^WOG)|IV96XmdckIj#O%p z*f3KHe6PZ2EM>zz5Y8K#V!p)5k;OgH|J0@0Z1_P?e=ke^fq#iOptZvMD~)zZ_E=YT z^QQu3YjyfAl`mM|i|yhM%t{0YU#3p$MW!4`?qb9_eE?0iX)O4RMk^e~w+(N4Y;^@t zk@!B|)@-1iYAF~Qa_8OF&POvh^FNi#gG1UXi_;v8w|L|hh+5)E-a0-!FBmP$%C#j} zIyw0r8Z}huwoeb?TrbXsbW`azUsFNSo*8_y+q{kP-sK#d+r<>=wZD$(-uhj~!}s6fI^Rw2UfTsm^S%chIS4#;VRsxE+n!9{ zcEkQJ&fY34>h^Enr8{P55NT;?kP;X=r9`?zO1gW7?hYyGknZkoK^l}0q`N_wHTwMW zeV-Nowc)nkgN-?^`*X!9!O)Y7!PK$)ZsS6n-y14H96Iq}WZXIBB=eeLh41~hFAz zw|X#9#Do4$lqn+R&fP*k!~Xzb+nz!M|3;K$?Y!!HHHL5y|Gz{T*$kY7OCI~g-W%`x zuQ@obu{DYfokgB?h-tGJqDUqLlVM#P zNTGf#SMWd(m&@Ba=MdrgNgQ4N)U7Vp2SGxShP-$^@b%*JSan5l(g6`L*;a0dKO_7z zVhDt6HsAvzV<5^k8tPzi1&VK61}eikBA=ZjEQ_EgbOwz>#fFQ)HfN+ISLv2+xH+;m z50yaQKnvxf!f_HQK|~1Q2&!!{!Zjb3Kpf==Lcs!K+lqk@n;dTF>{O9FjRH3}sN8{X z98Tp*XyNzyTTE{OI25d}cNm%l_(WhE9EP&bh0%uSFe<^Aj?@%-T|>c^IUydWViYR; z157F3FN+zTBUpD!1CzRuJ?W^(t+m{el;bb#iVGIxRYFmoJ^;x|^3g`u`)XDww6C+e z_eKMH(KmZgC1&r5_AOh;lHRzO-@d-gM{QXLD!aM{N$i!{cc7Sby4k|50^)bT1CepD zYu?YFSuMQUQ(z|zUW1?<;Jl)QU?aOYgRmcsh3l@(**?<5LB+p`@U8VI2l@~{`{rZR zX4lFinTF8H`5sp8OWvQA0t+582AKqVs_4Z-5z_A*Iw1)>}x?284q@FpD~n$0INg zMCKX_J4BMk*ZRLE2}(Q*6xRw$0)G%KftRoOyF=t|kO%NTL@y&8HvAS`{{Ov>v*dvP zmyY}Q{8=24^1t|dqpTh#zjd74{P^GT_pBtaRhni06nj{*1qR%|8J1kLaE63xEDw^xBUOCv)3;$0FkT9bMPF zVr{Lr8ZA&RGkDF<3lqH?tuZhix46{kd@vZ1F;=SHVu$`ek87bf;JRl`Wai_3R%??T z|7|F@UjKjhD6bQTSM@tUbl0_Bd2&3F(nuDJZZ|;)Ux#j2|uo?E7OLSY_irvOO$!O7T#bmhBmdsn{)ZmR!Q;;~~l$g_0qTIxtrd*{8v?VfH>O?tY&c zJ?>Gag|X#H;kM1XDPSFw%Y?ydlym*fn~bVndG_aZFKC>X>gH`Fv(9I?-}xTTJ?Adx z9aplytDiECbwrsIhzz+Lx2`&JSW)kUV!r~KKI^Dy*?r1r_>O&1QLyGu{Ic2Fomgjd zBa8)y?uTd{2_J?{FSI^9l*DxuB~$9=dh=N)YZ*D9%<7<=~qm2k5U`$gPv z39){?wQG;FnnFis3St`vHwwt7=Jd!7cRS=@_4U$w68ScsW zn4Pb*gyyJyNIk0V48QULUx({Wz%?FrJam*mCK)M{*mzsrv?Gkt9Ep?S@L z@I}-lCjK&`dCAl-W>OnLbWYbxJ}i$If9|6ZMHguHQTXmzA>s*!(%Ve=37=JSxHL*0 zS+i*kM|nQVCnxxsnX5aay{iBbim@OoDSY6UmHj@QV}GTrC(5k}-^g2jaQRaTSg$#G zne6%Qsu+k(rVdZ&#vMAhU}#9Ps34*d5j&@7rvGliW*M(aYTVX>FQaj!w-)pZi4;vr zmvlxxrd(ropNmviY#y3tQ3_JDHFr&$%g)wFP7Ye9eI2RuMKGm|b>IW4Yh>Q+Xr)bK zR+P<3{kx56qRQdqGAE#oj)T)wv)2ThVH1n?qWFyTq?DnD|Fem7jHb4$plnCPx$o}p zB3nT6>cI9E<13> zhDQf;jj-3PlhthIEJnFL4U6p{L1lul_P7Ar!J@cB{py^#jb*BB)uQr1|Y)S9rNeCLuIDImVg+0he7Qr-Oz9oJAIqqMJBx(Oe?NK_i)_qtW54orse__M#{jjqB z-}7f99UtnBetw(V;A`J`Jy#Y)jTyTcLDpMqUw)bK9qvTlz5pSZg|UZOo}XU>kAkpL z5*b6@!$TwI)FD{Ca8s?uqkF~IRzeZs+2T54tk~1ZK~NyBHIzfLkLitrgsjwdad!xH zl2Mqy#0EiA0E9Z}%)V3+0fTbQefW;*Jo>#OvkKsiaJ6HEw~7?SXtNYQg-E!!VlRb{ zO6c2Lj&QjWcns~G-q)MdZnh;G1W*+L;)F^ZTSmmKeM^4gE!d&FUxIWWEwmO3=$;>& zwthD3B|MvZ)$oai0tRU4DWkygW~3%>Qt5q(ChFWI0U~8+3Ke{l8{(k|Sx?gp5ssjW zExma-wds-)-pmc^F6-O&o8p`q@pCU>Sp{6n3X078IeR+vPzY&;Y%xJ^bqS2|_Eium z9i|YcUg=>kYQ3Iz79|uJmOvj3LnYX%@#VEOIIeLohQj_n_lXic6#T{fd|UB1a+29+ zow!3xd3hoCriP?aVJ&1^z}VrkKwN$q=b%s@1$Ru*A!O}hl>0CKdBpBuVn4%14;T^t zkq`Z%$!)9{iHL-7%$+S5m6lQ#>?7u{_r;Z*AEF26Y+XzrLgmE#9!(t;OzK7`yiB^avCoz+8i1jWZVmw@0O^AamArubY&ynrQJ=7yD)T=qv zXYrqgGJjZ*R#=EfSlDf-3P&iSBZ8FRDa=p~Pv8$v(h5)U2v183Pj3#-Tnx{?4bLHo zNb?|hqZ(1v3?PcfD{hV`UyP`{ji@GxR0I+Kr&r4ehXMZ+uIT?%PX1x4!nn@Aan7TE zt?9BV6U=e|5GFr~1^%lE$*ezGU;3Y3t)r7_-@oMKf8J1Q^{}$nC4PIgzvbj_2xQlv zHQnZF<1TBo`%LfOYr4PBf>;8Mpur@VoP>F`7Gp@9-hXbWlR2mnB7bhEUh0^7*ln=wVc^X=Sy`J>}iq?xvp-l~NcBs}M` zN`=A9_qJ1wa6GpYQa6H!*!ez1D>wdOs@~G4EPKknhvA&ber7v< z=6jzGq5oVGA7r6zk?79`!#RuaEf=S_W`#2rW~y0Rx%t^G@0Eo6Md+8fCVaNeDS8*W z|GBymdWRcRp>|_k-dwAF;L`MSzbvVBmH&XNO?=^?N(dvUJf^C>Hi~TmMfcEoh>hh? zw6L8^v3mF^5RGO`@j||_ubnF-DrOYDv}6q9?WkyFGnZl;bhfXb+9^pbg5_tP_g=oP zwDrrHDl~$Vo?#j;>DOa`i%8)v9h(0MMMvHWA*@P3DuIJX4u`2A=RzbTDBgPengzU>(ho@yk zuhtVMKBoM#z4+2dRIxX}i(Ak0Rk5$_=rh7c7F3ZYl7un#!e3aIB>v@C4n;B&Sv>7# zH_#7_pOIR)1}U(gHpfJrR#w+b4R)T}jQb`YU6y!%@(E|TCu56y0rlX#+0*M^7 z?{=THvMmm?iFV$so8q4q$wc0d;JDnCEk2=y&_W+id~Ii>mT#rI5%RFW=vXlWBx(7p zk0(w%x`{~Gd|ktYaYT!?+23Yf2|i)78l>D(w(HfLY*hcX4)patvUgRe3oXh}oii5jU zxf&jBNS{B}4=W?Z1rpt15QV^L1a;C3(v)l?{ZGyqXoAW4eslh-{Yr2D$b`k)VII+Y zoH&-u9|Y)DGAym^u@Yp)JtR!rGF*|N##|EBiOjsX(?UvS%88vNFZ}5C(somp+g4MF3Sz#tAR8|g$ zD?6BH`}WY27QJ(fu=Q<`$j#ma*38dOa1-%tA=(PgCXtc)t3{lmODdF*#2Dc79n?t1 zPCh#qnPWv9VTo2OCmaGNQdiS>HJ!}lkfc1^Ei(pPPqn1;iwrV7Dutm^brb9HMDE8@ ziC6q@^Jc!52+c7jS;~`V;3TK3MDL4-OKSXxs*O_O-BK5Ap3LS-G66+fsQU-0m(cLa z>t97{EG)mPJx`K%Jettki%@CGQz`P$_$>34Y_b7mq%f?#-*AGgp#9vU%H~^vD)yij zm4Mr;ICCbh$o*G#Z#1e+e(lm>dn=8IUwI~fV6c=57aodtBenW*xYu~U6ax)OsiyM% z5ObZTaRQuGD&w8kbZuKvib+nJ53+G6<<;Blz0T=)o%NEmK~IZMvuSZT^$q?#XjgTi zv7E5LMti?9@99Sn?G>A5*~8cYSz`gIl(E-Di@_;IQ$qp44$TEqdj$Pl=lB4-Rw3!i znE+ndME;TXr|^|&K3>1fmt`Etf*pH0DIF&+_OhrX>TY~-y%Q1@acq$p<6MBw@}$o( zKeZKb*_%J*1v84VXwEiOlJvJ_Y{<#^Jj188sY5aNDF8Z>tY4lQejDY5M}#Q%=)FtB@?k99);Fb|Dg2y zGwK#T`4GZTbu2BWECDAX`uS8FG7B6N?qPw$nNJV&COX)ypMx9cEo^)im?s2=x;nUR z&sUkK67LjLPOQ$n0S)CD_HVF!=KOh? zft}*%{+ki$j(qvro6GArq34AduG;+eTOFoz`}eNoC1(~?kw+CKdZY<$(#6f?4c~0l zw%KyohQ0wXJln1h zFj^JIFZ1$z_WHh-ebXgVZ0TbuNl7$yJNW7e5GN{0Nuzu>s{3@FmilXp7uR!$-2f*C zwPQy{`F9XWRVwafxs?`4y#|IF27%B6xyz+G2Jk@dRKK1bZ;pwKktK+Z>=jZY_ zR0(O)r}P_|`oGC1=0mB_s5>9lQT?Goy~j`we;upz`QX+ zgGDeC9WW_NP!dlAZ%=WFT z`=74|o&#`2(WslKa8*v96NTaKJK*Z&;EZ-7j~ay}b|b5lpx|1f-2iZ#9D+&F0&v5y zui=oPIXG-ufp8^QP%2zK*gq+fFj&@+lMz=_Hf*EXzc?UhF%Fl&1M|8Y^@anTUpCZj z82Ky*OT__Glpj6O0ayMM_c^0~2mt5&6c^6}m)a5y+Ko)mi+mkN((Zw2$RFNCFIU2k z4k$sn;ec4@;9TbfH*tg(uZQH8;PhJ(cZOk?HAAkcaGe)}zYZfyk>C~saG?O4QL!k= zu*m2VtRNEHSbp%JtaE)Xcmn__j*FJ`2z`)^OzcJ#(TYOw3<1nQt{8&OcOuSH!A)YZ zF4SPEw1DD(uqzMUVPM>qDluW2zkUzID>t5;H1LWT;%%HjmLk?W4=oObNX=z zBo%<^0b3tABUp{V!Qw_dlZz+*UciBb2{4m`Ae|h-rhO6Zr=f zJggaG|AJincI#iBfZs;`k>xgAZX>vbA;W2Z$@2FF@{J+1!#jq%5NqL|zFPMuVr+;C z#6gcOFj5p3}kVmZ`(75?xhM3|EzmAj?j=*v}1M?4^s6KPE7pX3gLS=FFyb?Mk z9TA@QSpoEy@8#Xk`Bk!cM^G3(@}TVR8r5exn3!X3m@el$pPr9PzB9Yr&d(tugqND+ zEJ%|rz9-%P9K4jTVu-i_#b+pI`bZa-2C1c)G zavic;gGrQ;aaZuOXb7yrCMLx0^=_Vx6@u?`*vS2C8f*x=wFZv%Gb;PuMx_phtr|o z5z5~i%(pxL++c>o%7H_b6}#i83P4lLK|lJrgi=QKH{>FS zQ85=obGVnN^)#3xDi`a$Y#-V3>2Jsdj{9&Q#rfs)^liyGgkTN9^VM(HV!04g)q{0{`VgOgZnQBh&}0uB+ znoO)lJDC$>H|!2s!`ClV@cz04fXDwEa#29XAU7f}s1~m#w;R5BTJiXu>73W zWc5OLM{de%`6ByvyjbLB1Ult+fAI+p+tL$<5f-7m%t2Esfk7_5ga}^d0=`j$*m%xy+}+E|ryO<%#{wVF=`8l6&7<0nqsRq8<;eoF5|_Ni1ITd=`%2 zbHmrij-+BFruVgg1h(5Rmp%#zqe|PvlgOMq+Dx9s(8ZRSoJ^}sI5M0B61c@W295JSk(VKLu}EzC>5L z6f1n&*STq39AE=^=)yKbsGCQ$2SvvNP>-egAa5ilS)No4hVnE84nEksQ>PiM_Aj(c zG95#)&eT_jZ`v=9PCh(k7+zv5@*x$KgS!%qa(5T}f0=Xm5t19Bw>9;STAi8`XKtVi zrS*+WbEr_PcA>f7_Tut5zVEjI0HHzsztqoET&R=G!!+AlwFWtmHcdWtd-!wlVMMx{ zn4Wd!5g<`lg}#c~$W1coP`)0Cj*Z#^8|4n=e{fRy{0mn>Ym-ul{+z4i5JOW>oJ!N+ zJPhQM#2!itW#B7vVq$d+7sNEflD(^Wg4iap{|F+1k~lM=eE2D72*H-0c7D2T$7iGL zpW*}ci*D&zO3->ySx-7e;6~x=^OI5gQJgD;#iQy|nRJ-*)fYBp>}4^M`9}?ou#+R~ zr2!;xuHyPgs^QQC%kIA*^*EO~pYN|bwN7wP50C&okr{eXvT(%~4Eldp-;cQz}sQ$Boq+@YIByqOyAs*nN?M}IxBjSV1_7;Xh zhXE3U=~$n2MBLsqXYuonMdzEXH7xW7`8aY1nsx}O2}eJ>PyDg}YFj&2(@5vRf_;{l z4>cZC20gb;IE3>OMACJ74jK$F&x*?Qd+|y@Qjj8nvCV|4lBAxU-vFTLLq32r-ZJU= zv>P8rJR-^-mTGQr>Az08hxxX(3%S=N(B(O;vf;_c=i-r?dID28sG}Ti!fv6N!n7*( zZ4%=YPR_b3%zc0cscez>R|_2*?v$e|#0l=zP{e)1SHm%VzubL`^-u9ge@WbaH*b+C zIHU0w-qPffhy`Au!RpLpviw|Hb?r`WZ@TVxX<{ufB{gBJu#X78k*$>~7B^=-qZqRvus?UV}>-(~1&s(J2UnizraeFK6_tK6} zP*5h+s)@V*!ELSnZ5s40C5Q5K-w-Ow`U&vrQ~TwF`|!haN{{Mx(gzF>CL*<8nuQMs z*yl9gM}pIA1=Rvd~|-La^5Q zA_;5(NRp8dI}9K#YbQ61r!Xv{)J&j$3sh#r*KQWkO(W1?3|LJC8hZGsk`VU6fq@`Q zjb;bUW<28(5z}45;Ls4~5Cn0n?*hw}F z%`8mP*n-H|pEx&+ayU#u)%?YxJ9rV0UJ{t;L1bqE%r6NlEFt(L3uIydmWBnFd4$mc zBjWiD&)i8INg`eNBgHqu=~RQ;heJEj2x3UUjvk)gX?VV2B1viZLmqhg{88gtQK6t{ z#_6b?izq&D7_20JxBy)Az+1KyS(U}_W(*^7^xbKSTC$9O)#J|S5rlCTSiI=+=Lb)(9@!M^Ge;$KJPuM37ue|9uA4e~ z5U!(@?_R1E6JFpFAlNe;3v4NaYGhKHjNq7xxQmCx|KU*Tqnx|#H;<1X)nqJ3`z$%* zl|iQJ9XoZ})zB{o(>Sq>T^4RY4ILPw3 z1oL@x^7*{-`P1_STl3ky(vj}PsE7;11q&o~3Z%UXWYY`e|FGGmBT}M+^CskU=1D0t zkR?yFiDz86{nk-+6AHhnlUY}5_SR~ew&tPrXjumol~4oBn4k5D7j3{7K~=O8zN*bJ z0?_VpdwBuH-9-r0&(H)}Ye7ngHN_A?fG4Qf7z)gA^8%>w3Mp*^JmU@Md$hoXxc)kT z{9K$(9)PA4PZS1G)Uwh%ekl_89g^x>u0p_`Ihqp&tDTP0BG+ewvuy8W;OiH!2nWc% zz6BI>=t0F;>U-FIYt);ni!ggor)Y3Xsh^c&P&6_FhwBBsx0wOzWE(EFAzv!0?{PQa zo@KXwUL#Of(jc4OF9I`DeEqD59M6oxsTXT4;M2_>ykES?g6^U{~(zdyPsr9BYn zs^xm2P)J->TQXA1{N#Z;;r^i~=S9vY0EHJS)Dn&~!LcsE*RG+MVc+O9O3J_C~5)REFQxd@Sj z&xyF}HhE$4d#^Ov18F@Enjk{WLAuQ$-pyed%@J+QQ7g?c56#cv>*FwiV4;?j1H3t} zmh`rk%$1hxG0NnJmb^Cn+{ottG?lR70B~>MWB%s#Y7FAVp=BH@hV-_U+m_~GVF zF>NyLf6Z(DKfPXmXO#Z^di{^AIm}d=LDGxl7~GwU(Qi}f zFM<%$PNvKJA5#fN5Q1gRcRmH6Fz@CB5n1o%hW*Z(NAc~$2tv{@Qz_{kENh--@p-Q( z(`|pRI46X8zoa0=dcU*;mNoxe-nhSCRt+Nv9hBG4TOU+3|NMMV*?zr$P}PmXa#-C@ zWOG zvf%lz`{Sz1j<@}-w=hIw=dbGn7@`rC^|Tv_*!HvsjiLOs7wgsGX&*p_^{oG|(Tf`y z)4_m7762uA=%GpAAoW4bFk_vi>_1o_0f(n;NnU`b`g;8XIXI*(kmFr;}^={d_CLhhsmxn6DToD)al z?2kJNe&KZSx{NJswl3;VXrA1=SpR_J>XufS@2dZ6C`*un0SoYcS3OqPp*B=lT>-!l^%pT1F9zrE*~C@mbK;cm;@=(1jF!ul3`& z8(}g>r=<~iqG2mrK&2kkG!3~b3$0R~@=qmuvf;GVrWEeNYjrZWikcHqrPgtp`;y8Y zhVU}{ISU8XzrUj#_8r)8*s$;Lg?$G>vu%!;CJH*yuN&svZegfPGAtKI*bA=mdmOzO z-5daSrOeSn1|ka;P+Uih)v8`_CZ?+3CDN)Hiuygu!#f2hY>@aq`5BinpaKRp!v;^k z7M~DHDIhX8g(@i`}Y@Bz`h6*4hQx{ zhOjUCC1d-$oP^vtbG5%9s)>3*<&_+_E=unN2O?-bq)hQ23#0bpk6Wzudb*eTwNSNI zZ*}kUSoe1S6S=_S{>E5{LlnDjFV>a7uJw5KYY#l}a;8(Yfrykm94xed|DXR!ZnOIT zO>R>mg=Itkbu|9L4)k+$^wSD=QoHS)sHo6{3I9wzVqJ$kPMKegaxHgfqsbK z9RM-(bO4p04TuT{0vv}JgA%CwQ62dNB(p;>2i~oRGd9Agz(At@c~Fw0#U?0SU^oyZ zJ$ODY2DPCfPhtabX&NC(MEW&Oc&;JOG3jV-GZOL4K`RllW4HjsbE`#S$W5PA7!T|C zyh|m18BQnh-Xw|M&?@4uj?e%1-(5^K*zYc4b_ncu_n0n6<~jeDy^;ch7uNsy-7R9= zFC}>aJuNM+m`~r6;qFe4FBh4xKB!te8;--m-TS;JMwW@9jQ|jfI~4oGHg;$UPt$)W zI-c!T4ni2+e+ilp&j>(?1I-^c4!NyNH=@-`|-qJ5Atv;m)G8{ygJG7VjG}?YPE$pj!Aqpswsj>V4QUb!E zyLcdcK_k@%HV_*t#m&oxzKkEn_-ND!qW4gYL;OL8#1n*gH2)F~j*Fn6N96q@?*hs< z=sd?)4D2Z0MIv}He#z4C3VvBD;(kg=TXqGLuj9!Uwey=GN9t9AmJELJ4Y6CR%^-=> zC6!UehbdQhg|}L+SOa|ml21yPmz&M*{f&)eCGG&P4D8EW(I*14sUy%z`4<p;P~9?4Ly8$1NWmZXtAUZW+q8iB8J=_${BaNiQ0G9bcOb-e zCh8UII`cq^cvL+ewP}H=WHwP%NC{i?JOT+Z>QD9v^shPthuSaQt%eNMF4FT8==lB= zuM=Mb78hO(l`^6+I>m1(?zj;R!iv{*+T;5~OS5uQB1(1Hx5CpxO|H->pZkkfL}R(6 zu(mDaI`|_2bqR?0J~co@5(c}o)Z_Qy2-kVbNiL}}#Wp&PP=lHG9Ggk!m%3_vx{Daq z>r(lN%`)#i=R%Gnd3jU$uceaC`MeWMCe{L94Wm_ykM>G&f-&AU19&~+X*7u<<#Y1y zr%Dy2Fy9(3G0Kbd+o(rdYom>_snKsCRZHwE9}uMT^{ABvc>ZK%*d#yU*eW!8F?=PG zpiv(oRpQ{9kv+H^-?~i53Lr$oWIzKp{oI2nJRX2d8C9az^HYoHX3>L!7dx5fc`3Ki z4B1;ayA0fzEoaT8`4kwZ_^#{BfYhd>R$7^PZfxD$&#`3AX=k9Lass6_`;HX%I%~IA zDUDno>@(B`8|KxU`;_Fbl0sMWd0(~2193&6cY3=-+mK>QCfAC{=_4oivh$21$F4T@ zlY|BKxeiUQsmP=>3mpml!^Q&(;R+ojb1{(fxEA)_$EAbd@1g`#aODza{kOW+Pn& zD>N7(4Hfj{L#hMuZNHg0iP_$t$9F)#6S&eSB45aG`=#H$IYaX+SzOn-<#Stno+ zU81Gh2GTbEP4W67CEsM5>X0Br^U-u$YoEeK|ZV>uk_hVFna2brH$nTFgM#Yi+0LbFk_QSKf)(vgu7#Qz zhdl6YYW?3`>WjY}(z>@P&Sn!d;_4schIo&k;o!H@4jik)aBWwWc0FDFCo=FpOW138 z{m-P%?B5G3kQ)*VPX(Rh5BjLEin@;|XP=&4{~`xbx=s%zTx!00+;x3&n){S}#qIt& zR&~vFDOU1AY{TdK2T|`u;;fr+WWjDQuK5~!*6jeXNN;?s`EqR5-NdV>rCfXMy`il8 zTsOa!noqt5Cs_|@vro`(JzXqksM)`^Z~Qi=3VE?=Tf_IRT>wvam$ObRCbO*xD4|&B9_%sA$+$9pY_2f%)BB`;!aJ2BR z31k6#BciC$2ACn{qKRn%tgnMCL+sGO2#f&l*Oq2@)9~`QxQbx_mD^ymB0|^#QI~tL zmL)I(5M=acY{zG_v%b2qZAVI9j9z zf5`8#A+6|nkLUqeLQ}24p=RJ>STxyb^d@7BCMY`DGK4!AA)=m?G3PBK6^o-5V4okL z!5EGRLeRL4K;YD_UW`h-jDQA2VoF9W-TKy*0Mmim`(X%lc`>>qh5#m8jh6%1{_mG+<p*u`REbf*YVBRFm*l8!iJC>6^1 zDMbo3LV7|p`WK=)AWmEl)&>W910w=mcOtPI;E0<+LEDfinX@<^8yom;Pn}fEgwdJQ zPz4#8W)CPom+Et#dKj$d`hym;CKZb_%^#AsqoFTH3M}9yvnWg|MYCm~HmBM6XhidI z>qWX!!$&e{<$@9YQqs3YOksQV>0gjD4yZHwq%xL`Gpd3!7IQN`4P?O0rFZV>c554h zNiwU9Gu?QV5Sls1M*P^!WhUhOhnLbTIkSY!vn+$NWEJSt`6N9SjrJNTlM*vy3 zN(S=*f%}XqMv~nHz(6Tk-+jQ8cu@c|9}#n*em+MTHitG#??dN)#yT>u_6gDbhB_X2!JcF%jIYMhccXu8cEff zNMMy6eC6s^&9`7>nl>f6v8+3n+Qp>WZS&fzt=cv8x=Xq`Y-mF*DRs0IUM#jkkRRA8{ ziE`8p?s)M^#tlxK>W)@)!h4Of@J)7zs3P?AFF2Z16o4MxSi@(St9?x*h^P%;KrR{C zB)Lt{YOEyPmK5)nw2XgV4If%^p0(z+VJYV}yNfsQpEXM-0~0cuL)73CXY0;JTA!P> zxt5VhO1hvIc` zb9Pv(c0e!Mogp1u(A*9y_*(0Y4kv_8PFTK6t}X=9$&uUnJf!|es6CXaD|oY$9lndz zyemPf%UitD%DB@J(#4kBwLZ{!bI|35;MOXJC3}VeUGG++?sgFGereoo26k-?Xh!73 zvJ>yIH|b#o_n3m*D^GDL$myvL>1o>W=zKIy9eXPsdZkn`pbou!;{+_M^e@Zt*vW~G ziZB6+eXkVzm=5u|efps5$bwOfuQTyPv9#XD_lu}vNMaERk<%+j;i)QG8o>9Ug9ct3 zqpQac7}X9iOb-~S+Eu{~VmkJ=2H=0N9e8UzC=D5;&mD|wBrJ^U!v(dn!4D%&4`TK7 zBXX9D7!OH7Aww^6hc5RR_dqw>XF35evC{8xJ&>}Ip!D@s!J4MZWK*ED_%@HVOZ%19T@GiFpASUa)86=yZAAN z+A-?sF&)m-a#p;tIMgZ*l&>)GG#r{)=6Gd$ecB;?*eT{c$K=%)T!E-@^VJE0hlwIq zAcP#Rl>_B!9e0rf`}_+kf5;^Ibh}vYICAee)y8-wJy8Jp1Sb6W^Z34(D!9K?aC=X2 z3)eC4S4S5fMm_Re1KKCBjr(aFM^nBa_?{w|#1TFzQo%g}5uyqG-hU}u?RgGANcI8U zX&n<6`iM*R8c-iFuwUL@NZyAB{z9yZ@yfTiQ)TwmBd$QjSHh!-*GFH0@UwUuBdor& z$kaVb?Xw4~v!o>r+KaG9WRn*cKT5+ZzJ59OEFo?((ua*jn69{=gfPSmN%A2 zK`VUQ#CpD8Fu*Gz^ZzHdG!n0^e~s{h_K{XOUMyA|jg>GZOz z@S2$FS|2e39L2iU#Min5vL_`VYRvk@*evGc`m?ijKz29Yn|2424dU-zqy}ARha1|j zH>!ZE=<%~>*rNzf_(+{TFOJtOz-yhkYlPFASsrUF$M}3P;0njF3hEzKpdVR%YgQZc zgz=ltMHt0JfI@E|;i}7>@LQENTc#3QR^TnPo(<=$AFrpkR?$eb3_A5D@eTZ9oZ+{- zskbXY+ZEQ^Ch&v)`aqpy0_#o^TO6P{TckhmCl2S1=JO5O$_?+QSw!PaP7zqnGk$0A zmPoo22IC-c_tUU@8W8Dx9-B=PFH%WIL7ZZ=i*_~%!>9TA5v0bDe6&+ctR%t~X$gJ; zM8BMeMDGgb?wX?!JK_Md-vG#8j=8WM2!IZTii!Mu4=Ppn10475llQfIcAj(YKOP>G zmL0x3Tc(XaKjHXOh%sT*hb4;Ine;6C?0RpZ?ML>*KnLGwF>ZZt&?s=P5JPfAuJauu5f* z2mS&P{=4_s>C)J_C?|Fn)-**I@Qh-GTl71xGGTV+f#3;{pbJ1dF}0L^d5L`~3>+fw z0w}Ldk1OnpPXJ3?jxI&~*wXs_)s-r9lg-b!T^R1p;!-mcIsXcM;H`f(L13Rs0cev8Flhdy=~ z0F!R%!DZjbJ8kt?3Q*H2w@CZ+R_djMEKeRpUn?X6^KZYX!{7$icX?|G;1)&qN znJs?FB?|?rOBQHNZ%Rdx@|t8ePyZMS28hMbY0qrQC841gm4^+jkNR_HPZnrj(Doy= zjP86G{UzB^k~*E@MC)o?Kju&?S?dXGoNkkvnzoRxOo6@6ma!wAu0Up zcrR|7yIgR&MGjp1zFJ(1#xhonL zXdRIob9_Rit~5<02#7QU63tz;-!}xNs2c#&`E+dSt`2lQv|U~6PBkOD>q!s1xKWUs zx*E}Q9r-+;Y zfwAR(tl^4uS8L4|&cFMHuZj~YO||EPW}5zNw%HUos&Iflkw0hoz1e0SA65KDGVRXv zeQ1WL;qShoRUW<+?*GNvTfarU|7+VY3|%ur_Y9rVokMpkQc5Z%Qc?oa9Yc46ba%I; zNJ|O`f)WY{O3yv6_1$}~UB`3Z&ws!V9PoKxuj@L`lEUlee=*zs><#}feF*;zU-8F> z@Mmv$qNNOY-^pI5sM zQ+phpjza&3Vl&%IsV4yH9E<k!*Ikh>Pb3U)D!hn0QtUH zNptri2}e1=G@XL;<(G_~)t9r>a$`-wc(OvjYfR@|@I@wa;^QE~jxKnIDGCGFThudX zVG!MudY`>+5~4uZ1X_Owq>OTK6t^qkJAENh9M6ky2j#^&vIk?uq1D?iv;1h?tOZlj z-4E!AWiP*mJo@pK_@0>Ie%Bi3Ov$kyL+1WmLE#K+#Z|k4i{>&ZjfN%%F0jB6X18SI zMe%{c(aX0(r1N|MYCvV+b7tv-^&Cs#hu|I?;ZHTt!6WozD;aU-1kAjGOPC~G)2{v~kGZNl3qu68f{UjXe#O!0I&!}~SVeyYgHC8nf z)*C+mzQyU4bA!>Jp#@?cWT?I?N2j^F$%nwyCQIz)7?QWZXqN@VI;)x(mdwnzepnNY zr-@+zam>s6L}KA}d4@7Xge(C}V>gqZet8QU4W*;#Vp{r>4nrImA$w@nt{S zCzJbqiF4*c9`l!q@TeSlI^C^cSabohN9PQUW`*gwD7Jd@o7+-EsGbSzdx-s{+K1yETB^36z2%vsHAP5xTuvab~GQ4{@7b5@Yh?Fp+0NMC#5e?O{tx;c+p7XIjJZ-O9A@ucm_IWm{>7T#W{j z(o91PiiQ5wrTQ`e2I% zlZ6w$1)#a?|1WG<+#_5==I!2oLT7cHz``i!AX3s_p^++@pCQ?o!|d7((MD6B$-6E` z9!x8;D*9R8Iv5T0Xyu(^OPN_JYoDd82OpTFOKG=H|(Rwz{CMfluQ0rIhRR=6(Wg`KPl^>PDPq#?b= z*4mMJrBP3$sqD_~c>?w7Y`kzwo4M^%S;*Roh;XIt`(rnXm9-Z@uf~=a-#tmFHk=Ga zol1_sdk(FYop}=W9GZWBy(6?)wj$cscKi|_AhmTwf7y@!WFRD4c?<8k$k6=$$($VO zBO*^sk4q*i#yaOC^$)sH$-I+ql)iRu&&TN5l}~SPF@JGB43z#B$LhPC_1&XH!GBqwjdl_I^Sp{g;?Q)% z*Ru2bg$eK7k`+VO;^gfPtvg^BHMdxCi81|!|if1Lj25St=mlEhYxqir^=!T{BIhY zM^mzr7dFPyhx<~O9z%IQ`TcbRn(XDXTL12tm$&OoPp8Yg zXntlty)u;To0)yNc0S~Gdq^yPSP&{+27M#`^w z=ndytJ~DO#`Qt`|1^Lq}AY$hYa^LUzx6kc!7a%1W+E@T!Sri|KjPP*)@mFCmdk~g) z;M?>-La!ig{-94rLFPL_3@Agwvaj9JYi2uAsJA~O5Sa*1KpY~eZz;j)(Dr4ivTO2fqm!t?gSRWTx5QO*QA8xLo zex8_D!DIcLuaMF)&oUgLish)7u}Jx)XrC}zRKL@zCE8XcX81gsr!eNpXpGlx3@3W5 z5nJpVrC2u0Slx(N$>La!fmpTuSQcR1BxOt^j?DtQ(o#F`az>m?Yn<48+^5I>yIAoX zdyzCM@l3EddA4}bp?K!)cuDMpFcuUoI6>%n0;m)gc9MWbg+S-vyk2&@{mKT&L;$Eb zx5#jpMs4^|Yz+>p`SN(ik8F6EiD3wuJIXgC53z`R;srz!nOCgH#zR6yS*bG<85@!w zTqW^QC1d!2>5KzElP7&eX4qm!B?_!030x)1tt6MAKfy>#matFcX!Ai{kfVp=F&Zc7 zQl;vPr-GwW_#zTuK_(&ypxH|5ldDtYG2m|${(J|;kq!5C7Df1HK!g?(jvJ{ntySx1N96rF$QiRvU zR)O*oApog<3>P_tc;*76F3?XG2q*>mS*R%ms*#yUlIQ`0i?MzWVJgW<#tVU^0;LqA z)szabBz*xq*QQLB{y;Z*I!m;10hpvPt`+FduJ%q5;J`=d4I)yTK(T5R&HAxT-2h6u zD5H(MdI8pN7eJd!nIA$zDOyV1g1F6~EZIeZxda_g2)^xt7rCHkAQ5|g? zP`a}0!V1k?@O44Nx*#G!c?@O{akVv$$ccviL8*EHxsnV11fQf(AR${n<}+VFct2^T z@7tslY4rjkAvxw+JXzExw~d6TgOuoXXZhFP%mu;Vfg%`tDq1)nt~zS*dm@)mOoYu$ zoIOl=AZIq=0(cgrI^n{u-cQN}ERg~csk;D{`>_X`#89JUv_L|o0)jdZXiVfwMsvQUV?aK_w&~ZZMOY@PYhN36?Gb z4eL;q@tQc}n)tbIy&0l5`9wLGr0ilCBZ5)Qp+N8FYU2a0$m7paxQ)gxs}f`;dNmlW zY-pI0OtSUXg$X3Bea)@hg{{=KS|h$KOqX@^4F*qXWTmZJ6pGv2Ort!yz>$k>4=>u# zxY|RN+G)Y<;fIEi`0ajO?W&9I)P?N`NyDTV#st$2=~o?;@Q$oWgPhl3lzdTytdpF- zvjpGZtp*qB_r0>K6CBi8o2_4O3NC!r)t=qe+11tUDA1z86-UxNDA_%1+Wqcz_gHrK zL|6CJTKDu#_sn5;9}Ro4SkF)Wo~huTrLLa!wVqEmJ^wH-O7cAN?LFXvZ~~YPZ+fSa zda?Msvhn+Vz3#ir?z`^lyFFw(S?P;|G~u82p_%o!S(*H#;caOGdFwHpO>$>~C9qxD zR=GG07ux#-K&kRv=L3`uj1cRl>NtG1m-3)$D-aDWLomPATv3a(WhWqqmym!rC7M3? zv>(e1B1S;R7E9WW(`zcvi;x%+Hyc*=8b%y7zz>E6EGMy6S&^eQ<#l@cOCVfqW2|;Yv>p9 zua14gEf~~2`c-c%+=)ICnez|xqSSbb*|@_PKVj`M@iK8j`*gfMhoX^y?5){kXEkq8_vDd2-rK2(UQN1^TjDCIsW~`7 zvF1b-_te-r?PNWH8v=XkXzIf{?eGaPYNOeIgtz4PezFkXO_#V%4)|mXGaWI#HAUMO zi1|d9_yToxK*Krco*pm6e+>F?TmRt>irwb)!TI!q;3-}jAJ$y~aUUPysnZl%9BtYb z-V-P`b}n(p4ED(S%*)gFyor;|d{}{UZ~ad2djn@(3upPa@vQl<{9SMud1jb-$XWd% z9h#t8q%NMwF@BoMWb?^fx9cos;=H)lJP5D!yN^ zOk6PZr`Psh97tR=A*3Cloi{(Gv7lS(=3laTLHURfEa<<~->~F#wxvmXO$D4uPA0_m2+d3c@um!o@@UDB5cVN*l>%SqBJDdsC_{wss% ziy4G8_Hm%Fo|OW+)gr>E9D*sh>q7A{1a+CH?tzJHOw`b=`3o#JnyvkW(8%d(Z1ROr>_Vtqt#2FW;7;-<6lyReZ9m z9I&gJw;O)ktNv+M+qnw>-_()$V)*3Cqku1uoyX9Ep{99Xp4@-2px?8S*|T}F_l&4Z zGhh#vxaaU`&-s4Og?>L^5QB_+A0PI`(;4KQw@=)-|BU`06xTmZ-2lM!Uon`@|L>;m zUo`%|o4Pu2LIM9fcI$s=>i$h}%>}4oy&BuuT5tP{;(GR7o)CB^QXS*sY}Siv>OS*+ z$sGD7lnL)%)c*5M=oy|{gB0jQ)ej3r2+_KJv71xG@TisQrA2$FA7$$UtuB$xcftm| zW;}YC>Ix)<%P1PZ*z^#|g}tFc(x&XIcf}eEhh6ov23UfUyFD{chnF#2C6Udd;Qn)z z7Sy0x`{Y7l+bfk|;Z6tRGH4K&JSY-1N)HR>H=40OAVaN(;pXoU&k_`%dgzb5d(K?1{(6IPtl! ze<-dTy9mYqw}k{12@tOm)zp0^j)rRLUNb+>_1;@C1jx5k?B^G=^p|Oc08k4FGV{IX z@IVfU8S7voLw3bP)@F8K*mdSN*_f>1{SvN*sITP`QLCWr5Kr~lx0ped6%|+&NOf>E zWV{nxQSB`MhAf`6iz#5NfW{{@sB`W6X+Kp^?%U zhg$h)So5(p+W;6P^Ugwsd|cE~om@ibtN%c)FWBchmXlr`=0#I& z<&D_+G5kQ{d{dT~!KPX%{vX!`y)@O=X*21S#iaLgZiu=LXC$ep3=lm+}-$(@ExKg<@gE>hH zF0*E)&Bw*Y6pBB_F^;EB<|waLnpMHCcj@JsnQVj=+`V43tz*Y&A-u3SkSd^6tW5s{ zXBy0n0&AW{|F-_8Tpq@YSFLKbn;7WMiJy^TZS(Wgyx*_XKX^vyt+ch^kuEId zx8>>f(l&g*KAPv@ywR3^`A~4LIpS96C}*P*(wO&ZqDy)7H@zFigy@GafN7OCr5ICv zue#2c4?X<P5YagTSkGvn@u%nlKxs#9P zxYLJDeIm6gUI3sN>&N=HeSc)>vEE6rNd*Iz=-2?!Gqn(U8+;rRw_phUL@>-F8dpI; zJ~YK8?2Wu7-saq3P(dJ6R569%c_eB7ISy=gkVYBFZZlN>5Jy{*z+f%5oY|rQ#Hi>r zMu&BtP*}4>h4vB8JLUDu(1{^_bYC2z}fFg(M?(oSHb%LZJTiljqIQ)3*+=pxxB3 zmmp&aBHc&lV_f$0swb&Q8Z}3@2pNvP=iePNi?5hPI-Y5nzxd;YXRYYPovw{^H{9jk ze>7d8K-J*x`a{T2Vlgw-8{m+4-W@G%oq_x?a%7-$dBpbcKC>(k&yW*1k_;!tOATE@ znN(jzD|`daMhe&#xmU8AP(P~qVM!WAzBs9}Su%@Yv@LPwOS#E#z6 z@4<0r=M(18bLgZXTyJI**TUbFJDdL*yH%vAAEKWNS?C^1fexPN;vyu#VpHrm%B=e`CY+IdW0^^p?EQIGxR!lfs&K`ya z+*O&JnY``6TVf4mNwy@`NFI*2Z-`BevK|4hj1Af}T2@xung<*6*?Tr|e>+N-!8b{b zT+%4v;ZfIKD_y8uVy^y@NZS^7J*#s(kP zSZ2C}^ih?wQ?DQ=W|(98n3BNRt3*v^7M=8Q1-CQzLT+Xb!So4uqnld;0W%Lx`lRgH znajWt6aP)xl#qbC^Q;t;@O&Eby@18kR4;yd|y^>(s>^IkQOIRGe8h+Z`e#7}werYXUY#^?`Wi0 zN!tUVv6m`i{6#KdOx?Sz$Wis7oAZ;%*Po%(2^~|`)2{rm2(Z`(oC%8igj_bWFEe?n zR8acHH=By`;%prsumjFY1@Aabdd5#9i3EywhAG<7#HW*=6qGZOGr9ST!yi2O5dY>@ z#cJebb4_S4O)wfdeUTh|wbm0O1{d?mUBvo@b`VMnDrKN_qBvKZM_RJQ@u&`W0}Kd$ zCmJBZp!-AVOa?K;(3|VUCIx3l5>F=#|J3R%+0zkYt;<&g>O!8cQ4r6>tL%o=i^7h$ z7(qCT_YBpdNy65|gz_T;XXQTWSroQQg`X7j z&Ye2r^x*LmWAfcTMjo>aL-V!A%35V{PMEY<8KdQ(Y#@y9$l2SD1&aq0FJ(v9gUVc(3c$Ij|l_a3}65TFe6;40B{V9au9WYkdiWM z;3)$V_Nb#@2m}QbsCc<5g5&r=zxCuV3YbR)d2|cZQLp(eOBi<;Q{^D-1c*v$hH|)3 zW3Z4YnAA4O!z=|nc_x3rVDMZ!DE$-BJaJsISEs_=n!y{2{pN3U2xz1Zjcu_VK> z&$eTS(_$yuSy5$OUi3IySlse*+^5U9Ez0=MV)46=rF-iR%pWgV|(-{IGQ-3@E2=R35eI)@6HavsE1&3m6TmXU_xTB zfr(0%iFERb1I(aR?}WBjAPwtB-c1_bwtm1M{u)tMg)g8iRn3RkVBZjCPAUkcJqasv zMamP0#Y7QY@1D#K7R^cq3HpE-#32#rk^Jb%3^pLyC{|yCl=9+4og#Ex-Kd<$DG!sP z*<7UMDj9_+ybP}&s-Fn@~VK#IJZ_ER4SkvGUiO(;8<5|6t*~s#243F$?s<=KM$jJVK zULQDbihl-04r7r2n2*kckL<`*4v}7fW-4oMA;EHHE~=)xzLNXtDtC)2?=w<7Z`U~Q zll_xfAC@x;@uMvDHPY_^a?zW$iME$zg@|{zsg6WF8Ehd*P&qQRsLLs zBtEAs-fF?)(*lqK8LF&ns8>kNNk*AfsM}CTYm&>bDosyabfQ+YE?C4W@ra`{i%VSy zg}12VBTzoY`*2b8P!A8-Ku}>{%y&&Jp`Im0O(rA(O5-C{xF$~L16kMSHItR-@|URj z;wl{!OSzU{2VrYT$kxdf*Xn}Qu1O3`icPMG%^gTBIp6;HiD%mh5{S;!UM-TtESvuM zR-@sqtR7yiMHx|0X|__?D^9R%79>dhX}~I3hy$c9u-JDM^agoC?6+F>+x;z(YdK>B zLFWmvcXY{*hvlaHr7zSWLY&yO7P#prr6{<^yO9dj(+aUf>{330z%9IVIsCT{pq2yE zN)uYtn?CS_*m?`|2^vnRZxxPSep+gk+CZg1VmVR{C|FK>fre9}UTx+@IjDSP=Vf;g?8&p01V->(=5mmlGt` zQp}dt&HzxI#>+s$Gow1-KwXDzt#)AzOHeT;93NFm>_x+NffD>wtGPEZA*dpqpawH& zfRpfWtf2&hTlnW$Pw1*hY1Sac(T!=kZ}B9}xYp_+NN^(&e^Xk06Tc)#P@`$ewv4B% zzW%(4Cb3LGqZzbYk7L?Iw z55K|MQQr<9Y?WO=y71O08}x`8tY%sR5h5_BZUef$_2qhDqe13x3JSV$dc#%5q1PTL znGosNo;TRu)6^bMlb9$eiID7o>UBJvZ3UagWtfs#aK-03b~G?`=96@mWP{(n?xaZU ztTY8zSCKm2#Mfn$i*dEoIEpsKkmVmncSw>p9D)QoyF#lBLS9?+9Fk;J#g9&sj7x&5 zO}g%=dn(l7`S>k*uR$}}H5jvvuO=T)R*@_sB_T9xJrNf@DwaK|m5m>jdMj;vg-*K| zVBOi*aw9R2mF!YN=Dv?>@!wx}jW)`iRDs^FfsYP*XiNH_dVPtDeKM|nEPDNKMo_-< zeul#SV3q-Pa6jI5|JTR?XUhSnF9Y{n95)IBSL}lfg#*}@J+){996_b;9QuD04`K!N zKF}y9wi$YWKKSd(jFq6`$IuXZ)Zo3wFxTnO1dV-g<6kjY_psCUAPa0b1_QuyZSHzg z-fTUhZaMsHyWXT)%8VfC@J7bEy6Bedot59nZSl}pOr`sc$rC?v)oKIK_^@FUIjs}P zi>$i&wP-F^99dVK;8;+o=9sAKSQs)Ef1<9Dg_M70n1x%;}Vb@Oyzfb`%BVOA`4PxmTzI7sF1qN_= zk&1FevTkEH9ZB?W2BqQ-&<2Zx00QrA^rgQwOxgMIEZ%~qoj_Xybe4md zYGdqRI3<5|3dq2^6C2`V4BlLqJ_?QyH-0APrhR$@nlS?(9I@ox61#bWcyO{GoTlnd z@vtU-cvk`n7Xq!i5-*v7q&ryf&6P%&!DWe44Onb|#$^225BPe?K$sXa2g}7e2vZf* z4wm~CHGu`5W#R!h%nDuIIPpM=9`+ZhT5=%18_a8|fvL@Hp*!=e`)S%Ix z@ne$tir`LKnmOxm&PvY4N**Ek^Sc#G`IVwOu(k0jS(;Qy+-h|kD|}&f4s)$|khK{V zgZZzu=dN}3taWd!_1>-Z)2$CmuMeBAzxyi&o9J1e`bP}b=C7G(ypF@OCIIl z(B1$>{!1+iZ@2J|GW(C6@qeyGB@yU2loU2XHG(-z|Ak@yYq@Q(J(3Y;_S=W$&VSdUawiQqP2$?wxaQ2Ed{<1Tsr?HIjKAC%s^ON;&l_lm)z|^%4z0rHX#a*(mh}FX+1pgnErSKQE zN8q!SY?o8G3}NB2%oxkIeN@@~wx<@#&bVWKEKzcN&auA%hBFw-H2RqCqL$m60RpTS zs9ru7vN<;idvcHP>D(5WRQmVtlbx4dvWrD-*T50$1ztqo?SHfS{FC?i--#KF;ZOhA z8UMzx>&^9`@*ts4DRL-cMr1LD)U#KEaT2IJC>G@M41@r{#CMZ4t=KYEl<89flhj4* zzocSG6o8T*)-JK6S+r7qQ88gXXMtP)!LXapip)KsmK(`TRKiPnmWwR;2Fk;A`Sm5v ze~+p(#}n)NLnh|?OoK-TjHIP`PRgA51fKNv_7=Es1ln85?T)e(RcbXBa|E?vWl4FX zG6&p$@X>G~sVg#CFZJPsL#3SGIp)`jJR*++#XirhgL*t_`zpChg5eLc4_{!CiLi7; z>82aQVIU6=nEFxc+V=bx+#5bg-Ip0yTr(zb9V8vI-t37dc0vXwkGrdsbP=+&JU0o_ z7`ZGjmE;Cl5&hkkUrb!iS&yinU*maf$l`Q4lEW+XY@7jP3?<*-qnt;sC33#M5?U}i z#Fy$g?^`=WC;4_&5XOSRH0_4}qHwpjJr2@XKk{p?zg}K|3jg zT|GUpMY~5BXg|Y3_*-X4&FlR}B~Vy;CJu3?yJ~{5G4^fRLC!duCy?VMBb@rMb6_laKdq}znA zxRbuhM<`xD%PVuHDo(|tHmX2q)9Y0fw%&hRuG|kGS;mhAN?7$BKDaeyC3tyzN=Wup zNLB8h=2opb`CRI3VRJX|Xz_W%PfP&MOxLe1h422EMBWVdR}2&t_t#(BWM7{5)_&-{ zIWJQ$EA3sXlbu`s@k2IBv`Q6uuOde)T^Weuk=Q|N-2#*VX*{If!eCo2oal)SrgTaI zvWfK&NZ5tY6BI_su@6F6*}3)@3vrDibBWTvdveEadzyK}sT_8JV$HB%L%Si`gBdxg z9Ox`^LcW<{kS(%W-vAtQ##2T$%O@k4p$M{Cjt)P1h!T(925FUcv^2YzXYmQOgwVKOpNj< z}rd&}m6S)bC?(djU=5hQoCl)-1Wv>F{O7 z98?;S#nr#Kv*}QOE}K?%xILx1WM>}fG>^e*?lm=}7wA=5F06UZ%p7T#k#jn&C@7+< z)1k~dwNBgc_KO=oNvW+)<@T>kwI{J_)Vp~)9i7$>9CK)7zV2yvRuw+Dtgezcs#NY; z-EltXj-fhvrqi9r^uu<_RQlXgyC=<(_hOw!{PK}?MExk^kWzqJT$L`Y| zbg_OZVQ$^9PGwA{)Pd_JURyk=Zj4Q>!Q{gF6K*F_0ygEL&Yk*?Qn8eveeGd$rc+DJ zY6(&{t&wm`*8{UDS!_|V!51y3hOSaZbk9>qeJ)Oof^Lmq*{Ne;$3ILGa~?B|rj94P zIx{acGv@f6I+5LSX3;=v!XuhCS$1(|Gq7&L|2%D~N%-gU*&I{hbW|SH_v)v^wwamu zXxj8-%g-05wC2*k(>{E>`00YaVGd_YpV<*UcL(P_Q8G%O{qgGD6K4KYEh2pm5#nlT zb0ki^uQ?yO?XfQvC#XBGw2;X3>YCgCiDpFV;_)kC6HDo*#!oYru3BD2xZXWAPssR) z_3Kr1kh!I0U&bUP2v!R0aBE8^+S#_M46PMw$uJ{$e*IU)I>yJIY@gzjFZTXHo&`WD;jj`R7 zAWtZiL}T09Qf!V^rW&g5H+UCNsq!FiJJv{;i9L{AwUk;Pk+SZXi5vrJ>rbDhHA(T4 zw&g1;-V-NM@9xszL5J59>_EjJ2ZgpSs*snT-S)u)~)=kwy7nJObL37WgsxspIX>g!2on4O6u_( z$9xpT76_mY1c;>s=$sJu7ziILg6ig8t%#~=0m!5fm;rJEH=^j+{J`)NptvyURVcuN z0?c2C?l439=9E~#Ko1J^MXes?Q}LPkNNFOmkro((^^_i?7$_z>_ACxAGz7Ix+lc^$ zhp1Zw(>(0w&S?+|K!hUe0nX>d;WL3{7Cgpcpk)_vWPOO4g;>sxe-p~}1qwy#3K}O8 zB3-=lpM%V1(EI`kv%tE*S;-?{*!N&UVnNUkEwf70`vh(N5tcNQ`FO z4ytUA-V%j$i}BttNAK{*!nU>EQbIm_<)0mkB}aE%Sl0WP5eKr2TfKDMxQvSj#D7+e z-SLiFe;ltA9{<@p{(#c<_%iP3GTxjufmc6<5|TitFZ%1!5sOs*x?K8Z8FtBzIKPy@ z%#6eELHN8yOl2SljO~akqKT*y6O|J^#1hG`9QuSq<|r5 z9wn?3-XJF`RRMnCwW731Z!pbHM)^D>Ig?2*6x8^bCa@S@`V8{?tN3U+&;^i1JD_QV zo@sU}7G@mW#hQ%@Yh%@T`>0q4#9wr;WRHw%wff{ZCgn^}v7(w$kI!=O<%4HdAalmK z?oqjO9=WS(sJDjZdL}dkd6mlo&fAU3gbC&CXQnWgKzGjr~FSs=Z zTR{mv$Q8V52ix-#!ipj(qrnU;MSRWR`IDk|$%TT9Z%kGS<=ymzI$s^zvxswgNv<+W zOQgz9FvB}@6grueI-jaYFsmKpXmB!X;jyASKm|CN^r&-nsF{s?Sqw~|53dbGQc$k*Bk6p?GCyL!2Xou=a<4q`|vKV8inb+-b&0TS`)GMeX=+es?hC97C=>6C{ry6T%ASQ9_^KyMfZx6yw8_(K!UMz zHKEe7iWp8Hp@%muQ7I=;oy=dulvK^wSpD@IW9C8CAUgT#g!hsQFE9gMXzxICz!s=nb~z0Nd@vF?Nz6NRND7u=(c@5G07TR?n$g6~~W8@b&;3uxpOs3+8`!hqvH;lnx& z#P)&`^yyMLcfnk$mrFxCxLG8$C3og7J z7i9^psxy%!NOkN`jqb?A2bWKDP!_hNoVHOz+O5tQvSL7$rl34W+S)Gdw~n+;uR$%6 zAWIF}ju=?|B%|96!_aHkNEaiFs+%CGo4m1`)2+L$3RJw-8MkK5I7pk6vl!F43>(6iAEL3!7CkCmp|QdJJV*#1-m7~1RT8?h z29>CWN>#&Tzm+RnARJM?5g_ z;`KYVaF|Uj)l)yHr5}uh0PL*UscKe7WjKoEHcI{(p*{Hyw_()0d(<6?K8DFSMi!hH zt_gBK0`YaXs2`5<7LB4nJ4vu4d9w!k%6cY@WRyVDJ z$FU2?$M8qB$jeMRyVBv4%!y-d(Uaf3X=OD*qBO8dZfG?(3~zMA!gA6qaSFY0if+6a z%J?2l{ynJ3rB{lij+=b>mSOc4>QfC$of>t^8m^puPX(Di zIq^c1WOJSDJ1xVBlhnf;(5M;7FHI0qD(3RY|J;exCTC`wdh(-pnYHVTHTvv2cxGhb zLx1PYr{}YZ=yS=Ov+RjMpHMA3uLKeLbBWjUQVi^Zc$loGxJb9%@^U%cQIUfumS!`k(_fU zyI>D0tKl1nW=?+*YnPlYy}hz!0L)pK6>(n+Uk25zFxGO#F5*6p7UzyKSp|XDi&bfI zTGkTin6tB1XXfMBOGt{x!S8CA&P+31bHTlFD})27PRPcJhk_BHQ_03$+4VS3nI74M zGZ$FzQ4gdceIHl+`{;zer3KT?!JX{&yUx; zRe8L%%dSB9;C20m3<=Tpy4yB>xEDx75iammfca}U=94_kfBLQgsAMbv@b14PV?_U7$yhQj zRT1f!!k59+|71c@n_}>2TiSTL=&SgYXXDzXpSBPPvJ#NB`G@!QOe1-zXExe-;!S=mzcPt74oz-wI+-I?)N#B72+~f}_k{s6;@$OB$+rWwaei`&|@6g@!ui zX(VyjV^|cvh%F=h7t5BFD)rmt?bzvY-I*8wItUz(Z-oPn#` zq#$WzizAtEpAPQfWAnlj~6~p5Zsy-zh0oCP2)r*{6n!5{LEbEcM&9 zG`2oV54S;wN6s~P&D)K5WTywp3-l~UP3xXty0fUpP5m`F( z_YvCkHrG+;A@<_XYPget9Tm;R={R@S7ddRC*vX4Yp*I|YaM6$d&XV0pb(>M7cyT_f z%2p$!$i;j7?}Nb?7mK07bd5_BgmD)GPu}o|IDr#hw7Ncy*8FK^=C%H7WwQhS_j*?@ zRCL4p&tTB+5c$w^_7gZeqXM8pe$nIK_3M}S%BSIr%{@!s6I=wFbH^jgVkO;75N1G! zhnGS6l}gM9-{bwT?U3t(H7a_c&l=$}^+&X#RZn-S_faO4?jORwKl({tIi3v12&tjo~pQ}xpte&N{GbiA?jpv!42}tKzhVs1;&+8!2 zg$`+&unOoYKfPDTh;^B!h(x{8V>(bqw5KI5l%^w-&Lycd*8$ABqZu6ll@KsxhM5Pk8z z0^Dnc+Tj_H8nI#x;RNA2JVOUmtjo#>H+I~{xkY5u7(wjY z0F0{?Le5HA_FFBCgu(+^a=u64@*oa{NJ=TB<@}pvx!^mGLW4_}8u0QlVE1y8ha3yj z=W!JFn<&cXZ%bmOzJ2wE-QgQfj8NIY?1%v0(&bp8;lcG=%Gg{10!Dv#|eLqS78=fjyroyKw%4+b!ZaL zOCEcRnoNJxuJwIG>SpROWIbt zd42u64~?ik5L**97!*r>D|JF}Q?=F3_c-dumld-dnu;Ki%R$QN`fyRHCwP(qgOWR0 z5w9qF(c;?0TSxGmvl*2^WrxEUdb!PZyH>oJ25zwI?+Tv>wmS4uqkI>b6;ZZYTEm+!Qj3O3U3)mT$zEeIElrd%cR?Uuu$#Qi2K7T*54_;rMbT*_&dp zR*Hk!1m7_XeQXsmZM*kayqCrFrwA2>-4kMsBh%0P{31Nv(1?I#CZ0az$!05@-Yukk z5dU@H5J~8)YZ`Irx=2GPi+ZA4HU)4WeATLKcZ`p`%3+g`?VPJ%JTEn}rPAh&ZY<7A ze5)~hU>JeB)bs#c;C{!-V5GX-r~Ruj`|b$vQ(_gBjBRT6z^q?(t?z69-X8AxPTL{* z$p>)Jb0f{%cK_P&`hj->sP_cZHEb2;c3C$`eM;jaxy#0QHLk(@(YH-XQK0T>+S>Lo z?+4D;sypwJweGXR&GF9$#EFYd0nBl4q!lA!sOMeJd8-l5L8@EStj5{}J^H=-Y!2Br zc3=1K2i$$!q+>}ohNBLzu`|ayd=C}bMW^0gbLE$}8C!nW4SoSgt=3QenSM`iAjaa? zA9WI4niyaMXXAhJ);hR=bzM zGR>4Nm_t?l;bS^e@6b8r5R~uQ>Z_NsSU8GcXqOT0u>9P%+~Y;N$IJF`w`KcNiipW) z5$^2Hy$~*+}dOZD1d0{gYP(tI6z#7x1=wGHhj!)5(Acmm;jaPGj3#=&z7>7k>N{H56_>==~yz?)9G%Mz%V^={sBt-)0Bs-=YNNZ z^yuFV1b!CHlEb!2wFn{*DPIUCfJY{VLiN6Gg;9R;CUY=pp ziXK1}cbHiI3Fg6UI0USun~j`#w_m*HN`;;ffJa)Uc);Df!vW*S(k(TT=dAQL`~EYb zFpX&UVLB_lM@1Fz8&U}YvyXFd3?NdZWU_nkh2#KUK6Z&RwhINO1t7yxvej9%__F)nmM-G$Y{nz{p^5j6)ba zud1>zdpP$M7CDD`jD0^l@)@{8$Cn2&TDlN@nuKPEW@TrO7y1SC)vn6+oT1k67(Fxn z0lbH=*`8-QSG%dv83>uYBR20&l+{VB1{ZVfPobZbxKggJ&2N}sS%P# zPe%QR#*{s_%@9=UIbUS_WqYA$YJGfRV2-@GShley_gt|v^735$3n=loDUX+n*;;(* z&-dYPuSV*OnGdzm!Ey>-e2yFpAv|t8Mhr@?rxCaKhXaEoc=OE?Q;O#{nc;6$@?4YMB;xl z-rxTIf&4eu0D$`BFH_z>oAm#Oxwj09g5STr>5idG8cFHyPU$X5=@5{T9%6>>E@_7D zM!E$-1VK=N)#u65{?RbyV@O z0?|YO+t8bwdv!=Ec5^N9v)DqZ=p-t_F)rD+hPg~)@c9g1Pr&9Tv;e3b(-1BBX{3^B z5q`$l5M%9um%6S6UhzMX^8QPG$frqTDL5?sCw(|l?-vgv+_sd zta71I!{(T>MbLj7xDgl7kHK1YS}$N};_pPzBpc zTfUMIIZ+s>itHhlIy+b9l;vP+HlY)T`O=75d)8qWaRl z9}r(3&ZN4AVFJ^K>bar3lWREicv@%t8S2!bhPXMtG!zo<_qvaR5~RH^FdJ&fCzYJM zeZ*N(m7J|*nw2MR5(?sS&QTi0fIEmTTTA;fc`_R>I&s}4&3LY~#TJ068B&7r%PF4} zYz_hh*&wsl@nm!>t$LqRciAG&rRrc4ganWbMEFK>MfN9Q+xVd zck`{}DAjgT=>{{|;^_rmEGoc68#noxl9d*H=fHj7i63nb$CX$73ujxWQo@d-ti}wP z1A9@V#d>uC{d)03SN|^gp@%Dl=0JSA7!%gf)~k+%LlO`F5<-vqp{E@ENAd0c^mU1Y zTySd`#xq4L8txG!d$9DcZ$IXRS!6YQezG>r0A|RHqWIn#hSX%v8d=A+Jjw%Z(81tF zV10P!evLagHGr-bY7s_qUzlL7N>61S;yu7Ek(kB#a+AwfbYxflQ{kZ2C-dapG0W{wsb}s7#fUxhzD!EZu%&|$(TCH- zgzinIM}T!t4pVRU4~_Ncso?O%ylJ!z@w%i*$(64q8t?Y5>2BtE!xw9O-s!0J-hGsm z_}ZwbJolQDXsJo`qPrLEBxT*-W}}B?_?5SAGWC5p7j(Yi+V7l2@_tuvb8)_{=SmU( z*CBN9M71FhdS&p!<@goKrym-%JJyU?LG%g~3)FM;~?Awl+Z7*PL4 zx3ClpC>%mM!IMFn5W~WdP#yUCPpH5Rggq?;=O#(AP={Zrq$ut7WRd1y%LH@usc4lX z`>udkAA#!0_HaS~dPZg6kkC^eLH0!@D@_8%9R_rsu#W=(fq5I73mXj{vM26g4xTo= zYliYy0^-Ed5~PAQApFyaL*bRbA%YP)m_8cPIXR9bsf?qXE_O;iDC^zrMcBE#r1-)Kl?G+bF!C7PRZ z8Q6G?wMHn$;W|2SEV?-}rW3Y#-5y;@qW4N3uRYTJ&qymp7}ClKKH(p2NaRZ8Oijf9 zDWC^KT1A%>p3%Nf{|6iLzXvk^Tcp(=GBX4b^U;P}8-x`$%_e^XndzVZ5zu>-nUft& z+mANn<8Ilb%zO;!wfwicWesn~{ zH<0;X1A6K9>wgdE!H`yu0lj}lTK)Hh&HpW=m9ojb>_naxU~Z@IFG#D(6xo5(6N0h; zvNKaWpR?0x@AUr;X=R+Q@$ZpV-us7bt01*8QV@*e!cRTA3M5 zk(`j3{UL(LT1iz2XF>iP9!gVEgMKO`i!EVni&>NkEX+gp5&O#>EKG< zo#;#tlGPba^o87SB#8BnovTno8-( z?-h&Od?FLBlp!CDDUSc%8YL!ItT)JV$m++8ESLrz-=F`-OmhN4)UO zNx$31NF*`9isJtAh7Th3ytx~0z{HK2uE+h?tXh}8tyUGyix;e0t} z5}R=d$BUwtlcQc6DQYWVZhdX3pLMtIr6qmV>PybSbnd4Dx{DuU*pX?iI)8ai0s;5BToR*3D#)o z>q^N5GT>g7WZm}a()SLmg83IWugDuem z*!Vvev{^nB)n$vsIr0?F=V0V@7RjeO^(S)m;!oYZdS+yd#E#eP$ycFW?ee{cfBBN_ zbD7`uLs%z2qh?j`Hx$tGncezMA1`tJ?##OJ!uJeN>n6jxEueS*1BHOc>n{^18Y5V7 zUZgod52PYjOuwz(MJ_1N=58AQTky{-Y~5H*FNJuEn1|xar2@{rSm-5L=5m2T6uWst z#Qv~FMHzB%|Lbb?Svs}WRO}ws;r4ppCKhgv^qGdVrdq86-{K+ZNxyUU z<5oWXRw@iOJKzcFFM5obPo(aCEYmDj$s@mHc|4{$ZW*~%!PYmcys=?siBX6`*z7TR`kS`hcO ze04|Tc6Wx3?StrblS4%cJ;g|_@x|_fy;dD@ph08bZ2jv`&hNZikG@0f&TQ4VuD-7K zrF02=AnG{V&X<*ai?bwnxu0HR{T)YNZ6J=>m<5QOZp;#l zu7YL-q>f)Sk7E1*wu-*28M6xD=ovGM>F$nI#dY_(j;2p?$qf*|oztKjtXsTdhfX z=TXnHYqz>eds@fQg`8Kw3;;iw>=P?eabdO1d*#n|UfeqEB58USmjnmH% zYZ_;m2-F%s($XI`%u?{iHGCqHWp9|n)tRcFN4IvV|BUz=yMEzUXieSXb+TIBm(!xd z+NJ%*xZ35d0ruLjpjmOqw~+O3_{Z{SiAEsPd4~fW{mpnW9KmZJZN5U{dw5_`O{6!`rRQE5q)X;?`s_Sir3?xu zK&Xv+^=M26sv|55Jt{`8{0X1>2Fb_(tNoR!uAK_d8rlzio_a57XB1h{$wSRocpQjX z*ZJy!ou@!BF5^R+5bfF#SsDS9uNojWygL7>3ODCP!^p@FlJf#usGr;|0!f&8%tFA| zOPFFTP`ne%BM6i2&%|dUpx-loOQeu%9ptW0b-WTB&8L; zK`c}<2kT@_1@w>JRKIQ#k`Tnv)6%uQv&hmYjcx%M@=3c}S-6ahq)6yuZw&p>?faf0 zEd$Y$;zGq8en@&eSeHwCnvs#_0dX$uRDyqWFlEP5*~mzvGuYbZAWp%^r=;?%S~(ip z73GD`*zb}cU;#%*l>`hqh{XgUJRICm1=JW|fo4mF@8K(B`~(gO$mFOj^~-Os{#hxx zq0Gx8YfXPlBjVYV<~Z9bDNl4!fDLk5XJQNOsmehcE17?V?94|2_A-J%k>qCSH<(9V zT$f>n(oBnE(~^1U44l%7sR95(sOtKW^4b0C1nJqh@Gr6-6jC|60fU77GFChktTI2S z!d;|S81JVCkiJozj%5jJLVS>S&l7ryjGO@;Y}9hrGE_}SjAN)oSood{*9RZHtDfp2 zwGldC^2#GEoqK9m9GwR^h;q!_;pGsrsOdtNpm zQ9esv27GL7QIRvbXs)%~7Ei)jpMa_)iZtD^&-9THRq_^~zsb8$pV&KWSF)of6qXLa zoP$#|%0L!SfJ-RX{FcDW8is|T_b|OhIG;T;LI@1f0uoV93CQ+Rv5IyUf?Y-Swy%QH z_TQQ#b+2~d=DbntF2^L;$)Pv7Kc7+j>){#$C*iLUR50vto%pLl{B}pN%=PY|?fBv6 z`+m+|*X`+7&|f$2zX$xym^3sc=C?dNet~K_eY-cxjG)d5oU^1sM2@6G(+2S1q~;@Y zj!9R%sR^UGpFomT8o=Ls8BXyg1^q=zKPr4Y9eZx1?^zhYrQbE;#hbMY+sO8t3TBWD z!txSSEPTMSp~_Bha`BQFJXn&Mz$6$jvu4{UO@gN`JIh)mr8`oTrTo%dbj=DxLn==L z-wM>cYb^!U~aD5SpkNE$6xGuVMKTARRNXM+G zJfRZZnCj4t&1!I4gj3&`8jy?eWEGCaKqW4!?mz{g6cTFck&N$kx}~F3|4sno6Z5-g zW{f=U0BeE0RDk7BNV1VQzR-E*qcN$2H06R)AHDM1m{i~MYRcVFEf-lBhr#&H^G;&R z#kQ5F{g%%2Zzjtnt{;ubyYshj*bpgHm6=fTivlz?hz!ZZ3`pdn5D*8EV^aATEpm}d zJOxn@6wz?=;PWH>1^HX0cC? zb4BEr5GA!r?Tm@JGLg#)&A3Y4a+Uc?XEL8MwWlja24AV#`u zv}U+9^>fogFwLp#eR!-rcbY1$nXe6&X66z`C1gb6rqEh&=; zr~nf3kSV~$(#a&EL_oTGO2tiC>gziGH8qRU@1@&tuIrA8h;8(pGZTRx&o)GKpm*E* z7l&76t?$&vTq}fEPE|^}cZQF$W+}c#Ng0P}?*7d-A<1`UA)5LMGn&9aCF z4G|s}V@MOweIrdH4j2n(6Kf5|8?qtUkw*2;U zct)RzQg2B6Iwu^zJtTo3bCWQjJ%Y!$7%3#G0M{r+niIDei(_#tW`$51F_SjY^giqf zGXS2u$ufkkZw={T-1G_b<8u!;U|R;_E8Z5Xm71?sXwZ;&a-L^^r*jJ^7#}g4LQ^PN zJDkV9;wfQal!cRo3$kOIcqvLmW@#;?cqOX8kkNMpQ2L>Jw>8;Yr}FZe;$ZT6nnpE! zB72?*Ig*zWP-@51UbpW0`RydWt$SPT*l+pg%FTzi9^Sprfs&2iA6BsX5Y_)bDa@7Dkc?L$oJu!)cE6-=`17)FRJa{uit1;{Irp2)&|u$=1%kt< z?o$ZUYo!efq_YCQW!lf`9R@u1$!YudtX~+>5~tM)LC@!R@Xv0IPn8OBgAOa_yp!-# zAI>XHe(f1fcYbdBa9QjBYu^I5YjO0C!?nFZVx<@Fx2QUC0T&I};Tne)@sqe;)fXxM zW7jlnKmbtyZucsx#?1i5!%3t@_d3b+KN^z{XPML8KLn?5KlneK7vR3zQPjAbbpolC zWhi@#L|n|hdVg7;@GGK3uRiX*87%gCd+0O$^OI@AHNAuQ30UKPodS9@rqOejF?|ms zuH4Qf^jwr{{Ms{t-Yrh|T(wRAI`W79T*d8$-i&HIoaI38cOH$&>4&Q>=&zH6-urD0 z=-slU)Ie|V!}T=u0sQUiVL_we1eXZlHcJDjkkW)R6r1PQ#AFtyanr;l4Iv~|U(o}` zB|AGb`{bN@BaxUsK@(sT1hTFHEoD6!Q(f1}fzOg&7yG;()}p{)^DIdXK1cLWl;c3R z4D%@uU0kEW?w7^o71h%T&|P4wK&A3blAffa;`br#%m~to1j4v``a;kAWkL6hXG4p8znoCQz$ z2%e0BN;AMAzP=*O{NcWl;5OH9c-IVE%t4?rm@)Y(Maa+i&&6M4m`e0VnEY>4qOvC0 z|3j7d>2U~-T@@SAL1?q^BwVS(^^u2gYNplB3$I9cpHMFy06kzFdf(QtF{*R;=AvBN zOvWn_<}0es92*Zpxd?KHI$*w{D|X?dujmw0g3FT*`OyO7A*5L?qapLj&Ub`iQa`v| z$$uw{gX&Oaw8PHV6|-^w-TC^ruXrd82lEwiTKa&P4Eiu%5g%C^5sJoa8AhnDloZ0K z+&qAR1j8YROFX4Xj3iE1qKqQRqDc&4Y~3l6VYW>z(Bw9|S&#cX&rBCX&`4T{!Vi+8 zivrX`=%Z!tm@N=gQG|=+)s1x-;&q9i4oZo_F5Ga&J-VC8i0&PmsTu;esR?>zx3g(T zNxttT>4w3RG7*#WFba2Xi4Z^#rPFS7te)FPgC(qq_; z4hGJ%R|yg{No3m2d}geUDyiHh>nUi5Isi#G{L}=gL~>}dom?Z`(6H3$J-w0aD{sC$ zrTjS7XR76)Mu}D!4HHR42=0#i(ps$C2hDNgn$A@i@F?~Obv+`00!o{)#CC=^KR?z| z&nq8x;u>LoETI>P!M9oy8asL?hW8YfsQe6!)-1ijS$Z$|Jv}Y!~pcq{(rJOYYr7#p&3P(E;TML$A=_1n)}&P9;jk?6XPh!Y4^neB0e- zlOop#=hITC>=&?jqxiFF1*V#dSyjQuc;hQnw>e!*@auU)yPC_t`HBJtN(qm?qMfOC zwo|#w^1SQlA;DLaTejTW=s z5>I)ioe(89$DRjlq?`twnUXU4*?4?8bV@b^(?+gD6Kpv|;dYHJz5>#LMPx27d@@V8 zjSwmtXBs0wWhh2tC=R3$Rh1n}8h=Cro^0UdMH8klQgRFt^q1!Zq(0^XG{UntOT_qz>u=zXbcrznS-1HL&K&@kxdkjs%mxrW4GW0`nRpg{8!9-1Igaru6$(v~u|u9>Tvl zU;j1Uc+#rTB6`Nnh|q_Ic%R9C;UV;_hoz{dKIOd~*Npd|q(8;Q7c!cV zw*AN#0qYu5JcXBEg@0X&k4cRff_o*@pZV^d01g6#VzvQzbHE-yUlYgM zaB$KwOHa!r{&v2uk8i2AQ`z21u=fc#H_rI#d>#5_Q zpFh9D;llC`upu~#`&BfJ_5qR)C!jy8#K$4H_7OptO7y?qp!thRR0M}WE0~*JKk+>w ztakcjT@^BJr~W!)PN$8(Z8MFs>0D3!f?vEm+3B@Jv$7@QJbr&}6=KXuE=ITyvb$eP zhm@(84-yv$T|Xy)(lg96aQo&1AC3GC9SZdmaboiZcLj8#PJvH%aow} z&aRc`x!3Q0{yGl%LnV^8zB`FD-&!Mpz9iv=ov%qYM>DBzC&GMAcSpLn{-P2Yt+b^6 zL~~MtQ-!OC-}x{7kL$O3srgNF%6s1SM>zxLaUG8k@FD!Wt7x}j zn0a*w%U`deq9#4g{I7R@Uq$2m_#7(BjNa#~bzM0w)Rm63PIa{%e+Rp7gE}c5!*?dQ zZ;izs!*>*z<}1x_F7_9THOhPD{H~5y17AV6^XL3;&Uf?@n1bhEpoSB{Iy>{Z!26#! zWYgu6&9~@%P~?BSiVEylgxnIfQU^;Dp(0b0t^n6SPrP^5z%(G{^+<-_9+#`3TzIIO z(watBUcm@^$L86(lx$=l9X7|xm`~{IDGqw&d@t?N$StUs95z;>?21HF{SiqL% zGGC@U{K&E|`+XISn&8znCZkO^T5q;tE@`jW&JVcSEq@EuBpXQcBKNY(4`!^~DT*!I zMQ4Ska_!_sd{$gUH>F$JDa-Yquq#RVp=ejiacEZ#qO|$ z)u!vPy~&&6d)sOe14qY!pCd=WPMOne=i%q~J6*@aY@Es$OZ%J!{p+1|dBYQT$Nm26 zIr%+^;5p6#Kve~oAC}<#$3Z+h(;q|Idd}{Oq)yLxGM|9APe+4^4(gaZ#Y7s1;CW*_ zDp;4FkBUHj*Lhe)d+)c3p&}E$6+%J(4cs1|XHRRo>d;l%R6+T648s)VP zbMTq>-HvmaI*{0lBK)~Zpi$$m-5$a_o z+>s{7qmmJMcyfyfb)Wh$9uPyxs^?K!Hu__3 zIl~XhX+1>@rC|j@J=OpmOv;@Bk}_7%hpa+)Cwo~RHLifCLuj~A|1p*)jEdlU3OYOr z8@WvytsoWCqM9JII-1afaP4q?f`oIqOL^{?;H4t>PUarY&;|`FBpk99yAfW-*yze8 zs|gN!5kXc!JZ8$q6BBSh6$+P;!~GO_>4Ji2D@)?~$6^HxhcSNIM!_&SDtd~nGQ~5- zG>2T-O+@waq+43qxR_RS zn5=skm+%>}v#-B5-%A$g)-~mJe>3@of%nz>SQ}oObJZ4kozzb+)%Y8{r_^lE1qQ2? z1X*s?F2hskHk@Ay>olnqmwrmz*Zm;;^h{lJk%rDs+Z*e^uLGRvNqah6oPbjIMwEz&GcTH9?2AeSkSQGiU*Iig4-_pl~k(5l4I z=)_#jMac03b*9YpYO=NBTv@A{zp~2I8cP%?S5~+a?%#Lr>B$EsR66pb8FgLj*No`Z zI=k$@?E&j+taw$wnRj~2LGrmbh^s#I8^Ip4y!CpPx-@58mxFakZ@GwqN^c3_6;>5#{^tn+hmjclEP#lrZ{!6U+Ce zG4WG4Ec$7B(&^LLwy*!@m*0x0_y%=rOIkpR)5;yOEt)}O=nl$bIpZH`P8=se4OPnn zks(MdO48^8u<)J#%^(do^p0|EK@)zw%zIonPx-0cq~l4w2zA7 zG$wl80|8^oT<7dtZ?vA1;)Som)JtRP+k;TnPm5#vwP&eqlUp4Ax9=?;X-?f+tjA~- zgO2fNpJCyu_H56uM2(WU=nsz9KjYT3_ z6RaGiVC9UZ+0ww~vM*%2gJ_Ycxu5#brSihabEOceAn=$RFA(=p7*Qz;;Hz!e_n(|t zAh(wBTlglLQsovP7PoSEfYqF>UKx)+u&d%l7l4$dXnA`;0}QEfnVT|&ct%3~nD#S{ zgMhsP0c=e`g6YdJ+&BOt#?Y@&x}jC1wp9e`GigMd%&&k6 zC>m;uG&XX88~mD>ko<(SS{MNYk-{`YN-R|?9Cuc1wW)?{dyXcSZ440y+K!0fONh%n z(+5yNn?zrCkMV^_qp#5b@$2#8l_Lw#_6p!h*4K`7AhL+2^`p@U#YkSJc?T(xqmoF2 zi2mV-7^d}6QVcj31zqV!Rul%N&;Ni`g?ui#yfg!qDuc*kckxUcOB{Uq#%JTpmR}CUzZQqE{~u2ExCp zAt4OV z!{hP=!;npWN;IxFtZs((A*7L^nE(Vlya=I4q*HKYdt~flWPHy}WFl!)vQSj2VN|+r zRAy#W_MgIcMMBXfe+}QkCP}Inqib)X>tRqPp_pdFm{#AI_CLaRi!nVnF@2=5146Mw zhOr~Qv16IA-P+j6#n=xwu`{G`vqEvO4cpJYaWII}QhVIjzs}fFz{7)?<4dIDH|^u= zfbqMK_@Msy!<~44_=HpD1YhZdOZx;bV8Rw8!4sJ9;G1wknustSkCa8S>k!T|QhoGOb2X|7iyk^B#x~cc>R)&S&8=GKB&|f5Nmi`^uX13c1%XY3~Yt&Yb z=Z{B8OXa|n<$J}lQxM)L%ouD#{3vN9X zZ6NDj1tdtBHK+`uW235m0E#9b-8ACp6w_4ZxLJ9fH8IMPg2Z-EKSbmH~G7WBK? zKkkJ?eRk4^_!ECzbYy2AvmZ6J72|^`v2p6;5&tGFx3Vgzp{}4OW z_L_M?src`P+|{CmWKF`y8^U*BPNS5=t7TV7O0QwC89AhttPT5*KFK9NbY@LMY!M(4 z#mV`z#`mk{_iCpPXO^*->!}7*0(_o{>#%Skgcj?wp9x3j*Z6Xbj(MMLiq4&C&L9tZ zA1Q8?m6|5?&bjhIrrW)i>xV1;LYU0u7zp^#bsX?ry)8#}U0nEVX7mN7*iS0zU&nD9 zW^LsapMri}Tl5CR-K<{Mh<#YjSo(Fo)mFzpdc^eh*O`w%o!|Yk6!TsC2pp&Ya8DcCbnX16j^j>J}r+b?MRFcNfD4;Gl z*nY>4lN957V1@qWMpkg(6y06U>KB)xBGnmQf`6(4RmaXK51dCtnDZLGo$r{KYdt82 z!HS-eObOTc)XWfTos!o<8Qm&5)g@qqs$^A3`a@FU0VW+QNk^Yf(QJaYy)@f6sq#}| z?er$O5;&JP?*&_>GV+Gjm^g`5m{D{xZ|Hxp2fC@Mcj9Y%#+LHFy&bwyK2KM+*rKm< zXmjoXfW98s-u5V~*mu+BZZI%iPrgy8EhUItrC(!yy9-tKoxsbf7u&`@mmiy|%$q8g2coCzxQTStU*PI~+|$d7uTN%W zr_f)koF8*ft2NQ70$h`+Hoo)Hao3wRg{N4oy8_fn&a)Z45!N+#OlgQ~*|$bFF&sQ1 zY6z77X2iYE-(Qr})b#bc9e>*5#k2HgV;wehbmN%@C}ny};rusqON!-+M!ldEA_p7g ziRIb5&&^g{E@j3s`3uB?WzAc*-_2T=wtf(Yb_H^H7UvlEXbHBnQyx8USvEOJ_Hdsl zC44=-VS1LGQM6N2>sN<4aFFcNbzEcW$JAnW-hAD2OjPy#%7o=^Z>9$h=VWK|y*Z*J zQ9t^R+Cx&))mXEuo)P(@5SFiyhxF}U(vf3c6U9{=QNvomh?}SKq@}`}Ofi~xE*B=O zm1<+df?*pcvkepLo)w{?7oScQSq!X5-WoQr{9t!{s=7(cxi`EK#r+Dg+j`Y(ZbbOq z$?J%(U$C!9n?ctqp4H$u!x#w&q(D4M>xaV?!D_B!Sz?Sg^W z-;do1Ux&7G6dJwQHqS^XAWdi)v76cpSs)I;gO-NLw6wWvWt#?HV-=5WFMFu;n1<|L zSo5%8Y8E-(CJR#22zvFLpIm)ybfB`GA-b7V7}050WU8Gi$38c5J=--Teg*q5xLq5) zNE>^$4^_aNToBb}lBw&y`$;eZ|>DRW~wGb}vEsA4h;NiKmzc|~6 zA^EEpxn6DlbGTq1PleV4Nu&``Xr#{Ros5G5QTuX~y;geH#hLQ9h;htV%m--7VKS0< zq8+Knw8@IiyA;dCmvzJBpL;;e0j~QR)HicYH7*okpc63XeJRmf2YOM7vkctk_2Zs1 zMKHpXD=F~5J=-P(vb@aw@h*IT$m}GPp&fR%1qT!Ucs$#Fe>m$6CS0F}-t5By|D>>` zy$~d{5P}WAkL5uGyxdmJAviZ708${n5RlLiNbCzFh2cnVLeL`pWMPeB(oi~~PzJ+L zOlE$aVy6cp78#FFj7%W)P6#_Bh})MJm6U*Ms#>EsPrML^o5(-gY4SF zpGVS3&4QfT!&^c@dWI1OhLkk}$VOb~Z5C+ek)QxKOh-46G=5~z7-|qtq^l;fsT+nZ zR0!QwDpIci$tV#S26asF#c+f}cDZ2-aswIlAuAh#t-w<9q1fdXxF8GM@)LTM#OQJW zZjKP!Spn857e*Twx?~ZGBOHTl2sX$K*0w=j(~K<#;OzCG+Ch*45@FwkcGrje5rA`g z!l*)nyT*ksZh^bjhrD(YQL+epS1z~KhkC_@e&&W@;~N(ViPGUkI)e?*bD{eb;7Avs zjQL{Pkm9OXpyd@KP0-+~tfF2^v8#@uo^qiF(BQ&wl2AsjUF(2Pw`K4d?&@BUNIGNyU_hV1FA+xPjo@MJ8H8aNa>s&O##hpl%o}I!z0-prY8?Ss8Ok##Bmn%Q|bhaCXUUsya`!H6*96jMFTXABFQ4(HZB-C>OHwR#*f+5Qb$SG%h6eD}-Sgu$Ks)&K6QF5pkau4&fEi85c476)|TQv33?cTPotX zE8-%jV#ucAouEbs6boe+i*y!?Efq`L6-$wq$cU848J8&dl_+JGsC1U7E|sX?m1vTe zY8z7`iInR5l^SN38h4hOE|r?ym0FOOS&5XrG%mCCE3?lobG#$e>nwZovF!Vwx0T@F zFyR6b2LAP}qq$lF`p>K5Da?Z8DvxiSzjx<KB z6H1$qXcJgT@yE6jkr}qA;NR@~y6?UJ*QF**(|W1DymgM+1^?2WH)RP&LZ;;kKM5%F z3|%PKnWlQz0jnj~%0iihBbk4o-}e3YDH=rCwM-wK z05qE$4FjxGJbf|dA2-R-2+9x`g$`q=IHu{AbTf$u+kwIHIo<9{HMIxAQdJ!Z*y^~U z?H_L)n+(15GP6wcKiT)_L|LwnZynFi4%;va-5>V-_S+B?R@hsI7)GIs{B7S0?Cli) zrPSoOTbj9!WLFm6ANe#4YkqhaQj+fISo~8ne50aTrP-mXegw+8U)}uKalfYhN9BHP zH_X1T>qGh5Tjv4N{~tsrqE#@vYNu zx3#>`Q8=e+eH< z2_JtwdoTMte1Q0(`J-6Z*NZt_1DBRQ4cS10X#=0b%LPfh8ZT4Ru*dKL{j;mZS7~`y z%WmcG1eRSoC}E{0198D`)?c7`kKOrqg3ASdTLKfmyYsOoPlP_Ro?{z)aa8qqzm;iM zd;25T=je930L*c>Q-X$FGHVdr8lah_^Zt zH)4eT9#LJitNR^MeRmO;YI%Qp-&h-iGrh&P)DW zhagdoC`z2?9eZiBnV%1x_C8YRPW=_w9SeJdQ#U98lNL}UrM?ZalMpTCx11a|asLoL zaEsVVTov7sic%2^3+J3H#CdIqgOgkrFpg+x2$hQ*qch5`ER~`?gtVw{Ru!4^6^FT_@nVecElJ*u5ps2&#=j~- zl(?tG@*1+n-ybZdSZg2dn@WmTX(*;4c{Ajp)E{*tG|W7=prAmWl<0v)M^{m#^xUs8 zVhatKQsERy0+IlHt4ha+2^m*YDNU(}-9jJ6i`Hx(q)e?Yc~-~VC)B8ISBqtd@Zxhf z-qBem<1_HZ5?Gb$sxdJ{^d(me!h72wo}kYGYpmjn6AnS{nU2h598Dc#*GZoumUZ*^ z-m#=Q=AY+nsxpyujejU@C&<4MD-*+3hEDr2Zsp%%?eM?NQnt9`OUX{|gqkdZ%QJKV6 zmHFifh&$4)mCwV~wK(<-Yzve1Co9Wu;ZZz~XC+s|I_$Dk&%0I43!x z=Ja<5mh*2uPF6(gC5N{+myOEa=%~d>eMe)J*<#>{oJH&=_)rh@W6Fz zY56-oa_iVrS9jXy+M7Z8ZLfsF9C@9zzKaMl&BWCR-if@}Q@P3RfPU@qVQmQN6VviC zx6oPkV|U5%Rg|}D*ptJ%;ej$1n<_-T4n*u=_nftL@4+(98AoYF)%C}oJqlwT=(0=c zOFnTgC#ZoFQFS9Pm9#@x;naM3e*{NTV2p8!;*n)YKYCTj&T#}9-FexI(ov%5!ja}> zwKc*>)<5sU6a3-;`;{sebiz01Ec3axt?vrSu!*`=#<#mI@W8FBNB+6eUhF2p1{o=a z!g~J4CA(;lQB!CSdLd@b#_kUzu*3(Cu;uDi$x-&1B$V+ImYSDD6z(7GTs(QGT?{^S zM{5-`^5?&~m(kr=f6sEk_f?A0F=5E~v%NYWrc2Ep!QP!q_YuEhP~a9XYR^H4Y5Ym>n~x7Y2ZaaKUK z)6gFh6rD`!Nzo<&OU+RB%ADbM4sl`ln@Dwn<1ehvQZhOrobwkc!Zm7{il#%n8p=}y zL8sn`0iE*-<*)J@nr2kd(QyvFS6FA4i>=qk!f!j>&iF1JsU%ME#1&S8r>@^a z6;G_+pILO}U#Lw>K$o>}13})l!gGpG0WM+*Qq|h*w40BKB2hGA;=4%#HXR@sSdh|VQmVo zf{{R@HX!WLGieucf*N`(5X`^h#vb9zqUZ##^va|>K*bjz!K{SMWK^&nxKt8KHyUcP z7~z8yPTA)Bln{goR)4wUYYPdtAG3KT5Xz7l#0aAqql8>9@ZVnhd$s#$W`YMsv~&%< z=6oXX+ad^{+>v^odIsfSA%!=3zLD{uaGveRgd1Q5Qxtr;KgST**ft^y5|%R-)wCVu z*Jejz|E5^T%eVa{d1SN^BpM$+s@xmwq!iU)sN1v?QXU!MP#)0^39R092Q;zuT6aY;>$aGQ?B`V{qy)TrbJo(gsM@yB4(uv zNF+mPq91)c`%a?2eS9=H-c?t>-%w}OH<0`bj2`Gtw5wk?28t#0#*4z-f2#L1Dwv4{ z_yqbTm}6I29%5-EAM0fo+rE?d#olEgP0j~o?nY!)ikgTy`6gN`r%OAeCxO%BT9R$RQM4@7)t4c8(aWft2nZAg$W=HvMMZ6IVrMHDF3o3#+-4&5CyBq& zPhrt{`$j)A%E*nySEeGGk&khqM>q7CkS$^xQ95VJzB$N?`;J&XHLv z;=2|!j3_=9Im($iEMK(cnf)k`Qv&C)CDBu}GO}}-WqLG&u5H4u?d<}ezMM14>wuyM zsHWx3ka&N+jS@i1kK3V|31M990Dm*G*cnf#8B6#j6zBehit+@vkOmNI!MHmpMXpw zsub#jQrr;`!Es%{3J%p~wdmizbwrI(?xM0c(6a`IP)8k#{Q;#~#wfbRe|qaUmiqUV z>MWH$zI9m3j0yAW?egO71xFon-A>Bf$jd!M$~}$Cz5U94v&;QD%VDHBH@{p-27X6B zklF<5Ynxmb$J|Ii)JSM2B*rh-M^h|Oq#~8Pf<&hxbEzz=vm$q?BLA+UK%@f53VEO{ zMx{m2Ul$xpfh4e2q!^2#Fjv&7R4D0GjS5$_vsM;&RweR6hTIYQ$gBHU@dk{mM>?r1 zJ4KqvD>!tx;Xehm?Ul4)RDFo%hOSq4XV;((CN={RRvas_Ll9~=@bYCC5E`BhS_%^J zaN1($QCa;m5K26o={33GN_NUZI!1@ zN(`mF!Dyz@5jukj*+6kEK#42B;&DO~Fsc9IS3j0ij{y&dQi%0qj}lCPGvS6y=7DKi zP}g>XJ7Iy#@E-XJfOF=Cxk%Fzz=fVN(`X97Dc5WfD8hmH0jDR5=XZeeMA`tD<3WR~ zU5s)Cz-@y=h8Ey}3UH9vnh$rW7HR0I8;BN9aR)`H#d+%*c+p{Y#FZN+XK~9=F_uwR zONuD^SzlvN2^J?kuA>`<;d|so3z)NkaU<1ex!jUsfg8|A3!SMB2*EuCv|WW@YtrEI zu(bnKntcoe%3`n+t8lIp>(aaMJNJ1~C~!{uQ1gDaorYkW_I2FcDN;D$4iVr^)N=;F zA;Y{5OcekwsGu#FuLZ=_3YDTAt!e}Acg2rq4ftc)QsBx1aM#>0L4C+=Zn)*k4R?;X zL4-J4qAgZ2A|RShM~k-8RmH`bR=DcAwXUw$g}{5dbu$mq~rd%*cw-zfK$^QTO*OR@XZ8g3IO-`Xa6|a z0FsF?gFp9HQ{8!XUHM7%u1Sr$YFBd%-_NBX6romm=b`qB+=%EJr1PGOrGOV$!j7*; z%(~w`&#rd5A30hqINTm_GaXe!>~>mccYMa{?>zb{cJzhfXpq=gDE;U<(^z=!m;e-U zOdUKHeLu!MI3~S4mS{STML*8F{Vu(GoImV;@%9#MZ8dD#F7A*7cQ3AmQmjC5*P=yR ziWPT<;7)LNihI!_#odd$TWJfF;*gy_&pY$Z`+YOtp4rFVJAYuU99b(Xx$gTq&k;ts zk!;FQ8(X#B(ijL8$ao(_OKSkQn21-8C`*iYrl&i$C&mChOYhzsxV{JI7 z@Tf`2V0_Sbd^mf2v}1gHZG7@!e427%R&-*{aALuCVkvuKrDI}kZDQkLVvBNeM|5(} zaPokXL)K>kPm;3Mee%b{$6;svpiH^pTGLb4>S5I;P+K9=c{n%SJCyaVvk=XsOBVJ%}E>0$@ntyFHuj4nbmosnBId8Z=Z~XYjLihI7g1OOxh2Mfz&Vo(ng6;Z( z{o{fo)uQuX7CLvoMbCe?&^;~&QY{6&S_&~*3j51K7uC5Gv%VDfxCEtIPJFeTY_yz8 zHHGK1oau)Tyq}>y0}R%acJTf$01h*qdPfJ8U+QZTb7r%kP*eAOv2O66x%l^0 zaclFzzg9i}AI`;{QhM^#zC2B!T>bhs1pxC)J>0`rJ?(DReC3o=~f}NVKgu zs@$WZKz4)69a(`Dt7)j{!qVG#6tdghr1i+9#SjFR;w=kRi`VayP}ZNne_LpWxu5=Q zl29kryu`~0iu0XsS_dhTl}W}*);vN&YN7lf7kPB6Q0f7rcuL~t0eJvQNkmjA5zggW z3cffdB+N_JIV=qY(w(Sj#T$ESi$iqSP6GK1g@J|WWEfv1Dpo>pP0(J!PG<5+MoOlQ ziw;KLl$Wx(lGMQ9W%q0iH^pi7_T!2wFA`ctGbv1zt8EDvHf<}(5w6N=cQ?@w&~jw2 zH$eYJ@?yLg%;c=VS1;;Ur*nAi*||fca_{rx%ACD8D(mq}wAy~5agem4f?_n959v70 zdId*J>*P#&Q8V=AQ(}J?Rp1G-8BLf)e5dkzVU{eTcFaVi>~}8#1=UsH zJD2J!OY5=PT&=2dBqalANq>Xg-Vc)?v$FOPo^f50>OHLVNha1(0eG<2&x4%)-#CkhwXlD$Gb5nOci&_713^9}Za6*Rf3hNAOg;+pKRJsp zhkNkgMHM3He5Cq5iHvh_2t%VP<~#RZloB3cc2|9DlU9kHYTGdWY*m~$VSU$Yla?ap z`T&>@$C5AGulx&9_au ze&);EC}w=!%c{Bs{grbw&MX?x22=C9b4fc#vE==xpZ%vf`P0^t8&okB7!;a2nLvGs zGddmRtX1&PSSCVsM-+jd{E~3KO#VJ{BB3y;$oTw7&VfKP>GSJMB(GzcP}Zr^GS`ny z*5%4729uD}#w>oKGHfRswISTF3~|3>aH7pbzL;yi^kb}QhRsZndP*kCJj3gvm8^2( z-<8S}Y`XTN+EboHRaV*+S|Xq48ye=y+LS8|lh|}*n&+#9LfP4Pg>|j`gKNK>vYL9_ z=?2!k_~ZhGcY7%QrKA-UoAM?0}gyFF6)1f9`wf4bGOcbs=}6}Qnxth zDx+!ul?$bC)ktiYe9ALi{AwHE6e0;n=yC8Av#$qa_cZx?eOG(iz9u2yQCF!Q;pC+d z?sv*SebODy-G#kT9OVJWclU`u@hE*^Z0T$*;55+pf!?O>k`128iJpQ0QqZD?C!=Aq zCrCIFgH9X9kC>n@qwqlfZP!jd3hNapPHS(c_c6sf61%|+&&RVW8~5CH^M1`jY_I<_ z(Gc~hBf2OQc~Jau@C8cHn~>}tY^39}aJTG52qjUMY3QkVW)+Y&O^Q-Z?0MLGWkUoS zX@L6E5UG8KK2S>^#>?W|U#TFPXFDk86!?QSdnOdZzW2fISn6E=+e`@tS z{*Z-{FU=CV7e`mC`Ou(jo^J4ex$jK%4A$OivFGX8yftxifs1)mT2}I?H#*`mk)QdfaACLUyF(t%In;(H zUK`+-sp&AhR6qf$y4O3Lv}i9~f4T|`_oI7#_^}7~?R7i1-=_Cwd&=(0wcTJ2oeRJ& zgEP)4eW!C>RLytncH_oM%=na)Q>0-^Y@+ey`~4w!?K6L4X===4ho4*MSSi3O^n>wQ z_Ro$(ST0Zj&_x>>UUID6b+zge`a9q4QBt*QzjBH?H>3J-kk8>^_t5ydJ>C_J#CbpZ zRje+2(;N1i!r$){J?okhQ%s3b&S`; zT@Lttl_1^^rG)3`wr%^MlkkA=9;TQ}0nshGur$?Qx*k)yfwQIVb1xy;I{_pZfn?`! za|dJ-Ey%Jy5S#`{mJ2*K4^)H%vb7R$ECn6C41{g93 zq98)OB=&SUg7#5;944p5fp*)_dk;4Sho{@|Rk=ru(BJz=}9)Tt-QQY7tGZ{nO zYk-;+(2vE}wJgegII5Z@>_9i7K``)Bt6%A`YCwe7=NrNrdA&ub-4{JzO0zw#ylY>& zTe)CxrDx2ugc$Bn`!FkOWQfC1N^qlK?3Qli>xfw8f>tk)qKx2DFP_3Fo_HYkYB92-Kfd2B3YJcQ$O(IE#I z5x@}wd@uXkvS3pZ5M&W5cn+no22mZW>5#-QEh%G-n7@`wd})@*tN>!oP-BN-CdM(x zomoW>Soypl3nMx*B(ZWcx&lZ?K<81Mv<~5Hwis|^c*ji`3~JYN1K>RNSVQ-+@`zqY zkJcCgy(A0P&alT$NZz|q=5BkB!ki=kPBOdY(puIH^-R%kOBP*@54%p5AWIYAO+Dz3 z3n2;p(U{6}l==cO&A%KNXzi+Gk&3}<pcrZth}Cz?vv!vmRxiZ#YP!X5iOfW; z&zfOON9HxE71A7fmQ|ty%s7Uqw1^EWctAZ?S<(-;OiiMAZ<-c;9<-x3luLz{K zDR9#(#I7iyImsWl&U=koh~Z>~;T`wfrjY+cGvzo&KC&o(A%-9AeY-`Z_#KG(vy#+G z4wZMF5LQO109yNkgxE^)6=^ZeO3urQLh3uqbynb7{s$@nQiF;Fb0m4>9t_7!KDA8f zdIlbhwxl}E{@Af(TqoAXGxmPF1gQsObk1-I-Fh8brpOx5081}hze>t%V-Ctxm>_!R z_tMUq%nc1Om)C$bPoiAPx;)k+EQi^)Z?L>|n3-)GTar(<7pk-YQidw#q|74i^Om<- zzE9%8DxR+Fx~qg|MZLmR{qU^FyJ|SIYV>ne_Z`3|P+ocHBXzx$$5z#LJ^Iv1X#iX9 zJspyiiwtrzIx3`k&!FbOyXGjf=HzqD!AaE#8}O98_CmP!SCrgL1rxI)5*-jJUlACf zQSAY&LlUV&x<^Ozsl&*s!)mX?S*^pnuLDxlNCxvA%+geMSeeIj!N<83(7Xn4k6`r-wcH%m@-Uj{_@ zNxZ#;O9c2TlUneeTI3DG34^GrGm=1|*l&@CSvx+ zclS-c63ya1p8_jrSk9WkKvCv=M4tk&l_a{5K4do^U_=p(EpP)>%!J_UPNGON2myN~Qcz%l4`~Ah$+Qx%^cUd5 z48!;y@MTpDi5FAhoXw*b-2;BoP@>k0b|kQn&Q}SK3(;*L>53xNe0v6OBMIjfKmpv0 z$N@AwV;ME&zB+3Sl4?aNKm(g$WQvf=nPJEk;HYo4Bhdj$1KCS?FhlW0=2n}TFSX|7&#-+*3lj)+WV z$4E-n0Mf5vGtADyHDIYBO)+Ikj_vbG%H(Rp=e34O^~%pb`6f1rK5ywr(CL%P>lkSp z=VeA2HNtB6M^>~3{=!jxt|jr$tcY^LPig$cEKr@6Z>RDXLJ8QWkr6=={d`poy%~GXzt^3KGjO$ ztCeD-l@h;|vYeHQ&Xub5m72$uI;z!%SF4RitIhv0D|%e*rds=x6&bA!`mGIf5;5eg zAyLhby7xM+2Ki<3lX7WDc9aOv?CbB-h%)dnhL z=5(EGeSGyy6DU*#I@H|kRhePV0zFq|_(>&-m%w@axVrziX<}P`E`L>&|3GN}phq$O zQw*Qg8DJ=$Rw{~8p`rMHsVM(TZuK|v@_*dtFd0h;R%+o=X>8`{J>8)`lJ0?)f92>FBi5gFBbb8P-ph_hW&gT>dUyWfav&`1enn zPK~=2hBAR#T9|T8U{%b|~_#o@AKF5D=$B9*u9#d4vCWIoKcuz)9X9_~qd6he@ngv(_Dq0V894rn=PMK4R8W|`l)eQf67 zug#1A#~(N-8E59>g-D!U!BLKE+7S9$n-nYU4~vER$VmU>BHJJaBH+Q#0s-UgpK}nb z%kFl71%e*jRo3{N&|q|odeL*@!pT>nMK`qzFN9%Nk6*9HNqPdNIpX@6FN(Vrcb*co zt{svU(p)!gE5q&Zh0>@#Ua~Tv9d#QZ2apfEZ?qo&i^4jZY$Nf{!`i~%IxYeLepny) zVIP?7^Lqj|f&Nn3m4>~}`*>pa*Bphd5oiOid|oEtl{?_U5DE0k6T7@`d=oyiOuu<% zaQ^gt_1a-y!f<-y<}0(ZDWfxtXQzVDI#0|u$s4WHD=e8uFlmmcUl6LvVx|bdYTTbZ|A#wM4Jzsnhtd9b?@T>ZEa|_u z4`piJ3obIoetuz%5IDx3hlFbG8Xjls)Dw9g4%k6>;#HtRnKwf|;LQsggRp zT}MIYV_s8M-;lr{xsK8q|Lpc{OqreFwVdXa)C$HtPyhGNEe@Np;9hFMHq-NvC9GWU zRsek-yac-PL_)dEm~`V0H}~Z1yug{iqja+Um3Kr5r7LEFzslXz`uN4~4|3pv(?!&#^Ns#X~H&^AL zewkq`X|OQuO7(~ydwu^-K?Pph0QclKd+;2zcDpUpIoQTs3FvN5v_ zqm!%EomsMI-4U=g@|`G3<0=#ZJouL8(y~V6K7vucu!QLXUP7s2olxzHaNch|fFrwsMU=w_SiveZg z@g4aPE4$XlnuQ1d&H8o-#>oLBxLJ4R*m&+z^4ynlCJ>9{$DKlSUI}=w&Mx%^e1*{g zZO7F1G(d69IVZSPvi(xip;@nXKeg3TM*RtFWoZdr!@AWwjgy+PFoiM_iFC+4 zd(QZHaU|9M8C-Poq!y{T7IpQL;_<_yWa(7Q^!|6TOB?XRyJfKQr-ZH)E%yT^<{Arz7|bz>oQDTc+?M^x-y^ z_S>=azX_$LveYwrbj4cu>uRF(iv;9_jlDgM`od(^IHehMUr}(3D;qis5 z!Hev{$Wr()lZrxe0@*^^;V6-{Drl7abosOdO=vG(^_oLGa!^Gmcvzi^9y&a2S{b8B zuC=M92w(COqBqB|86|-~;WX*bG4ds;`#K@RDQdwSruoXkIszvnB@R@Wi0JRLMBR&P zP3}tJd?;u;HKV!p{8v+MRuHnYa*h3fmN{Lbd%nIprK;$!Z*>p$m|+KI>WEh%(+5oN~5*Hf{3lA$-yj( zz+M(PsL252;jkqtTocu{(7GKy-HnJUom0Gy$O=7_aTM0yW0S|7*p_mZAJ)EDVU0!? z`6)AL`LXTES}2&{r}&A@+$j0pJ9z>X3*!{B>p^q_=E@4mptqgjCs*>K>qYZG8~%c} zLlj`yn>>Et^ErMHT{F?o(DvOW6&-@&gINSSzy&EpokZaUbwn%){;r0Y`M`=-X*A#I z=g7=&C@(X=#U@?St#aKCd4Fb4z#e*&B_^aPv4DrD)}wA$I5%29+7MKtEckp#X*}|F ztEULpD=2Uzr-*Yo1J$K6%tNZrW0ll~jA&DvbLe1HCWChmU z2P*oY6AkBweht&YNKE`-ne;{j*{WH#pAOF@81-XJ#J>PC|A9U__x~-klEWB8tN&R) z(Vwu~a*smYa%oa`gQ&{#kxqvhdp;KbsuEUAxxmq`H)u@Q^sbm#3Nk1W2>MVO?8{-r zEypRJ;E^Q}`~z*Kna1(hy{;~YB~&+Jk!IGuwT0q4Ebu|;$sfvVE|L~`J-Qv9cpHY` zPgj;N^Xy6@((>~0S)Q)NVOxV&K~)#Mb2u86Bp7Nb%D_~v8`1pX*1Nj#iMg{BW1Z|C z;3rLg!QC?;<^FSc>)rmfnA4L)4I=2(@HS0Onu}bjXuk(C{h(p&3yk7KWSaNY~ z2;zwF;+A){r3Y13gz^yt z8iF9f2#{{NiuMwr7K^QKLMV|_=zMo@?Jvg{Wv;%OxCWkK+UQJdOTY!Fn9O^2TX}aA zq_EA#Q019Wfr9XnMW9-`yW5hxNw`gLgs0q22Cd9FC$B8q?zd(#<1IH8d#$JWQ^O?oVT+(M) zsXwp~JRLbkZsRtTSs{}VpnBmWm&X@3LwRPPOpthI^n^U71a5G`vw{TN!vxNrgzivM z%yQ`bk+*zM!XZi`2VEkAMx3Ku$TRE2B2N$h`WQXf4Zv|ojiU{pOGxXbP&_lHuS313 z^e}Mw$mUf!zPN(ESz%(3UecZ=u?QweT_ILeA*g?tSSbTv1rwh<5*tzfgHa^D$uduk zQwoK4Dzj56bx&$3TH3SiR6BEKx#MUlGQT$?DPAK1K3>Gm3iz(&F7Ab?A&Y6aHzZEO z&JZE|V1>v4ed1VF{CI0g6G#wVeLCOE3_6Vr$N*9L2!Xb~ayBM@u0nVMYeqyH#E>`h zz{07LEaVL+NyaSQ3@szrIa8~j$YTVUc?;N}P1eT@HgXDWj!bjq^=$b^#$6L^6GQ+Q!;Zid}T!PsGo=dr@!GU(T`usLt4C?bS}H0QM-@l;#N{Rrr>+|AS}A44(USf0fJ8u;MF zh<*1uXea-cERP)Ih&AH?8p<|8%$Kh(xJk3!6aYp+fe4F*PwpX@5F{4UaAL#&(R0YW zdXA)-|9m-mYZ!5IroEt#HtzE8qKH`pGgQwIP_8HI314PebFGkC}i&h%cfG&?H942-vjsDIi zi3EfM*S26t05PBy72XxGI-r2Sa=7|1x zqD+%eHNH|cc~><}UOmfJoskKmcdUk~m%}8G)-o$O8!Gt~L0!T%@I<+byrPQrBRUA_ zbhKu9r3Rz0c)J|57lq#_@ZqRJk-DJxVWswot%&@r7M>`hVAp7D*QP4g@TR; zNHjr6&qI+~WnJj>De`Bm zdY+U@w{w)wC+`?VfV>FkEVRXvcYsX`D{5^}^9;@YEue@tke{7G;tepx4C4^|N#;!> z3L+i92-d3)GP3P9SLb3QPE_P|c~`z>{PMoQ8Z|Kt}KY?w6Az*-qPa4&XB!z4Cs{rK6{QCCh@7ITl^eKnNvXVJd3K- zQOH_G!jTqmJYR<)gN&Jv!{CCoFNMT?20(+-Q3FXkp{OugGrJuRF`KsP$=5G2hnNirZ?(5H%4zYmJkh^Xe#prnG= zxZob|9hHIz;EO)m1zyl)UoWbej4yG8=b*p`UNW)gH1J(bkyB=vt1zY^kWUmy`T(L( z?qdgi{_3W}hu8z;?Wgi>-p zw)R!~piFGXq2|Cr?1U3);XC-TIm|uV#WQ>8HSf?*Sz2V?4?&{CA%??YzQYmO!%?t~ z;h44IxQAgV! zRSZkb!)O@(Sc73t)7nU*@0icWG2iU5+S4)f(Lr?d@t*N94drnQ(7XR~`|x30XRT{6 zn>kxqXA@_#U3Ag|e{vl%UKlfp{(kb%aPo&T^XY?a{2(3U8c;Et@kn$kf8WLs`VM8g zKiiq?K~xRxw`^I0C>k&L)|XM&U1~&gx~pK)(W(DKGAjsos!7)N#T+Y{pJj%uFWv$( zNJi7<;|yks3AZ#WmR|!J+N{iwork13K5U(W(1@5d8z|3b5HLH9uJM)9(ms%p#f5iv z3ia#xILW%`t1E3y1Z*tV;K!|oXuX%n;I=} zJzt10oA<3+$cBG9pZQBzCnM%IX{lcQ?SeT(+B8HY?~dqi<3Iub5=d zIs)w+G?6N)maam`PmhJGUJa%_5-3p-wBe?=PY@7w&&i#y+EA{&npx#BTdRm%dzpZR zQn1F|vu1y~Hbu3r0A6plSQn;S=YkciJCCmmZLf18ZeTy#5Cm^o@0tT(D`vLSA=(QO)=t;`p1vi58mZEudDY_ZU7rG#zrn{6>ew)CsE_F+uQ{^bUiy(r#n1|yR>w> ze#*Pt_M7t6yX~17(_Qh&T~(DmnaQQ=*gYD=J!-l=O~_97-d=dqp7I9h^(080 znn?3WNxw$-mH+;;^Zljw`)_MN=3U)p{s;642c-4;^rHuM_IqL+hwoGm{VWa{w-3I( zKk!xs`KrqMKON$Oj?|kDLSV)No|FBN8+*}R%tk+y!hT2vZye$A9upuQrdl0aJsmNR zAM5=%_QF5z3_0fZ+sofLR*X9;$2%&FV=kUlDxn5ts+LvTgFdP9r}%$!zdYfa_(q!W zO?~jB`w7^a%hqmuI-{{RQgb@AG3QouTGMq>#`PU^e#)$Jx{&*-%bvMDS81sW*u+(_ zYJa-UrLph-{m{7i*#5^H_4n@^z-|AzdFr2QPd~iuf8y}|#92EXnEc6Q^z&}x+s)(= zavl%?_Vo$x6q80_MeGb0@B&c#6Er1_J@t*~r!G>!*^<#2bn+bI%MUKp3yg$we*W_- z`-@$%6h^hRWWV#4nhPG-=E?K;i*w^&6o3msz)#LEr`!&|zIXk)-S{O|yUdq&BI$4@ zop(v2b}6Gq$X=VGI(3=%o$JBg?-a8xiSi1w#vJ5 zYq+u%XSRo3IpJS-^{BWzTw8KqIK!A+zg(NnUi(j(d(+$m{X91-OWE4&vcSgNdSr2YZY8TVs?b#ubSFun0vn2_WdO8-Xtu~n*(%%Ob>oL z<}QZ;m`LpGW!BE2FbCjnk}i^8zuUzF!}WLl`Uj91ze>`BkK|Leo@qQWy;Lk2>+cr4 znLdY|CxLA>TseviRAAT>E@edQ7B~c@?^!W#oxEfO=$UG<-pr0n4A~3&kh+;&IVhb> z9Haj1oQfQA8rC-}LjfXKLQBE_{E}D)N5t<5>LzCPytMaOJ~O2;Y8dPYfI<536XSaC5Fov?>iEC2~^H%e_2cy$Z#)+dhc6Cl&REO ztgqNvFEVMG>Q^dm|}YlwG1`c84<`nH|K|@7zwi zD=iNHFND1%WbT+hH549Sj?SnQ2wk~n;Gj#79cjrL%G6yZ59S{}u@G$AfKuMF{`Gn%4Of|sxYaBEo?J|KWqM4*3xSQ;vD*UbSm#XM5!%8-?`^#Ty639fZ z>XO(mF4dW(3A9}`WGUP(HRKsmTs0Nhn=Un#`Q}}4X**Oap7q6PnK?jH>L3AiLSGOGtq{&TAJocXaYtg-xS=^v4sVhZgSs3Q8P zRBO}U&3!8N7nJg1$SJX+p!ll4z`NC%ZZ^8lT>fYSUX~`-ywhI@nQi$?NBW`INu39r zM@hy0M@RZTA*NJSI{o&YWq6jj$sZl5bs>Qapt#_!@qel#{XbdnRb(UJ+t{`QzrKIl zR1^Mkje-Y8Kny^+BH0W?WlCSdf!#0z1F>XTwsmpHQOt}lXTo2Mnk8$AO}kb%lf4EDNQePXu2OEOZk z68%W>{&Ed&y?4sVv2sa#^(IvvQgO9vT+Pz#>C)Fpv#_yk*W#o14HlcT=>xak|D_F3 zmZgE~NEIj9Y2i9juG_t2UHjYry}56HcU)Flb9Ykp!L?Ahu6^BGqe$Xf;r-4ybqY3; z1|JpH=Xa_SXZ<7&|2Nuz^d=0#r|Z2mxHf&7xTd)`eN%!48shn;##gv6SU z{wgdwsKX(Ij)DeDZYY-MId&jdOaYFiyi|E(U6>%ex$iOD7ph+rE`d`BgoVrZ)03Yw zVu}7`y`K){j#P;$BrJtn?^O@Ob^dH)3pU$3IvKZ~?U3w-4)6|s3$suzB3~RH6!?4| z>ljl+c_=?5yrRL((@{iyJ^Z&^3a6O%d(sd%lQ||#xtN}0WLQ@CA^{4w-ZOiF3RL3# zlFb(|`P$@Ba^Cu=?-jG)dUb1@lqQ#m;_zdUrHVfU1L~Cb{ok^Vea){=#S!Tfr;Jd# zJG!8$Io-FiXC1!+60kliCSViMiTEo!f4$0EE;$AT|H_;$Hl}Vb5KlAvl@&USyA{<2 za`e|?JosBKWo2xhy*Dl|(npr)1(!>Gi#bdWEQP7D%@#zG9a+i)O@UY#m-zrjM9J{r zpfG_;8od2O$xpiwsQe)BS@w}NKP}P-f?6T2m4uY=9UvuAg7zV~Ox8i`Wy+Hj4(NPD zL02)m|CKbdfMCCPx&{!LF}h@jQXRzKtJU#P1cCvN!O7kU?1(A+4SW%^t^D=nV!tbd zZo>lP}^19aszw6U7R-f}?f2zI;`BYQDVI>!xsFvu?FO_p@tvhCr>D}B^ z7h7#>sRWmctz0#?PE^}_jID0mUHt{#4Idlf1g3G+zWPfpbwDpbJo$*_TBNjoVsQO= zX`<#o(FPo;)Os(DZCw3#v;i2ZbwMQKTWBIT|6;wLtP6YjpMZDkqjdjR?{5Zp;_728 zm3PTk;o1OddJMsABu_uLp49PrbRvVr2;sSxxX%zV4cQh6Fa}4+I?1IHzoig}@{jsG zPbbTueA6=*JNnwI%w(QFaV#p`4*QAMN9IFf86BzY81P4oIUSBA-Rd9kF5G$#2k%~! z$ok`5lG7DgfGO^#BUBp;cd=+uUU<(y;~I-`^{tT5Z+m!!=Zjy)>Pc8GeI>>F506#^ z_{;wvV`;VjSH{w@{Qr}&^uHCIa*!0z+=h3z*5HoRll97rACkBzj{;TEj`oIKaMQPi zrvqb1KnMfY(@X1}-H;0`=Xz1#nY=F=GdH!$o!wq`A&pKmnS*I-h%ad~mrA?9=HLr0 zZbMFu3`MwYVu@O{LvXT>;J5W_t;9`nx4|MMns4oOGg@eTB72?FCjCpuP9m_zm!R&; zUq@fHs#MQ@OKp3k;LJU$A>TlHF^M4&6SQZcLA<)t)+a<(rcPj^L8v8EMH)F6HXWEP z1%!(&{D5YjTVa&VM9E>lLM`ZIO0D7FTF4+-Q)zbhqn&8aCX9p_REXz{5&sItZk*`V z5gdwx_Fn)%U+eq_C=MI|`VXePg))%E^B~hUb6*0!Fmn7g%Xz`tQVSE~<3WyZT&hH_ z0OHDFeu$3iaxN;O`;kQaqYyM7D3l2-1Y&$H{RpGA(aePspRgKO3nw$>XKIF6gL90@ zPrx5F8`&)L24N@3n81|JG*YQdP!p+kra4w{TBk*B<`?JB&|)03#hJ^efx7r^ z>gRZ=1!_h(eCSc@4BV+$%+gH3dY|A)k`Vo~AOHJpt<2-&FFP+ByS3zY!thhA+z5LC zP#?yGMZyrwTom+0?-|F%I6(r=Ov_fn%V|zf5r3N4bOBqvB%-C_Y&So%T|#3Fxe>6P z98lcQEo|AyJVj4+n&5+BBw9XeTI2MJIUBY+;Z}|HEpgD7Lx>HZ;roT z9%KUigsRAK*Jo;Ttt*7(#}ie(kQ)~L_mP?pxjrH{e%WvP(S3O7^Y^0_Unu`?BhSUK z9ANjCtD?grWX`0v)2`Ox=c{XU)W^r0epV}Lal@oqf9nHy(aCkC^DijQA8+ITQgo7t zbDh#B$TUQ3at=ZvSNG_9v-Jiqy%9qZ$Qn2Ul7xl=vF=VjpelSl9yIS^7pfC8csE-u zStyKa1*8M+^k9@S@}YW1+Tntx6D@AKBg%Fiut6B3uykqe_~bejx&B1-FP8#PMv$WD z!yG;ZQa{VKBZFFT?2)--5#s>6BSppzII0XvcXz}i?3~}+aKp$;eH;phJb~%qHpdrr zbV!K$rg1V%FPlf=F@)&SFPwr{8tMg2iw*CjKPIvxjhG|RGgR6fFl^YKGXzy>&d@f< zcW4*K-Q7=xc&$<*!3G*v2h_f;1ab)!&IamN|K zD4OzSUBleH9lPRiV;41IeVVHagXs^e@wDdrD(^1_YuB(Fp$5b9vK{-$Ay`?&l0ZnksdvPGetl% z=R~V%bY8*zcV!WBrA{rI4pL%M6_O~s0bG>A`Y(E> zPFw3*pIao8)PDCazy4a0ZN8H{|>|0g``(0G}4iFLPEmZdE9!^M2a4E_* zhT4)MkvnD(ceWv7Y z3KQaWTfpGIFiZf`eCCYwcTJ(^Oc06hpGj23{z#|!j-Q!vD5LASHIgkzHvEnw>q$OU zjnifzFmv88tm#_wJ5Y;&aik&cg@Ns3ymmmyV%q{%wFlCvsmc4L)Go}DSi2N6SpW1m zv3q0D;i)xx`N{&9OLJ_?u{o@G$+y?yT=V;(9kMrPv4BVA%!Q5HH*(Hn_v`iv{GU1e z+8gpcSKyQCq*o8S8#i2vg|9jZ1JMoD^bpZnqu-tdU!zQpxwm>>+7Z5zUt_Vjx*9N% zb}I5Tqs86yR2AmB@Nx;=w1;u_zeTLtZ?7<92#shPw-oyYnSMCnpNe`ONnh_n-49nF z%Z`nNBar|@KrjhmS;sh+EWXZTz2#f=RJ|xWhH#MH)bLlAr}*kk(UaR6`T2yP4=t)J z-;}Ps$8&0~noI^h3M}(aub)J(zkU3H9^|94MmisG`6%jgZ9L->5>yPOtYTpi#Q;vz zVG%N(leWl{HN!vTr@r&F(!hh40lB$YO?#LSQAr62?^(4&WU(d;YMm+X?1yc*x3Tlo zgga?@)wuD5;1SFr4%|rUzkc)H6Y|?0AH;urX2K~d9W>%=gZ7CY38H=pE4ZhtPx`$i zu7KrGsl4x!lnxvI3d0nYc$)Myr9HIl5uZ~7ZZR`*#h*V(?3olly8z=-P+`#g$d?ig zWQL*jpv$VQMkR%v(`=++!wC&Q zTBIN7UX}bnoFt)GXkfi7(r-sbdsnQ)Uw{=So>M0tqy{aN1S4c#%v;(Jm=6Sj#E}M36ZBY)LtkWS0P_V+F3q0L zo6BPJMrC<|P_-h_K@n(XQGj|sq*R6eQc$O$lk#>XHnEyEcM*GcB(k#*k3a@R5`w}>rD$GIj0`AN8x+TyDDWa4Wh7o6F~L?c z0bcfaW|<(1#zL+xPd=S6L6S%>l*p)`$mEsCl99;PmdLT3$aR}Yf1B`JD2W6yNx&;f z5SEc7+?FI7nHV|yY-cfvKEqR5KUvl*Sw16Ku`OAKHCgpGSzTYgrzn93Kc%NXSuZ06 zo>1s4rx+h6_G1XVUZw-zrdW8TT4khilcg&1rE(RfI+CS1W2V{agWteqt=+zRhlnm- zNi5T8fnG^kUTLBuNE|%=kc{-G<8XDMbb|}ufD2{>D1yIy2Hh>dYs5F8H!XTO!(}-^ z6d`Fl4T07xgUm?|-;3COna-mh{Q?N6V+AZtOOPX=E~#hs6yqo&B$C%5wK!tG2+YbA z$~IX}xb8+qI7@^RPe=37dRg7f_*GFLLGBHil#*G7-Pw<{0A@iTLOx2%bV345U;eI} zvL_jEWeT`IqwQbLHHic~(E_?>63O$?p0)rdrWq6ALO`&9jARZx*+Sap+!x9S>qg(s zNAaCbfU`MN$8o;XXQTFuR^aOUvl-5-w=Q= zwB(kj;O?yC3RoJG2^du`xUT!?EnLARRIrVJz`s>`g-|x-fu)Fm`Xup@$qV#muZ-$K ztx>onl(7gI0tkba+(;ll1ZHp7m4nIYFoA#zQ|eouyyCkm)?0vK9e|l91JkjjYDEGw zAMZiEWEm@m?fm1S0DiLpEo8J3cvtD$&2{4lI1dEetD}HEE0e3E+|dHQXBJF$3PAv9K%9WYkek zmrLQL8k&(>rVk%|^Q%&}lfv?=Mg!}nbU`wXq?#E(1=tc}x)lhNMNOyQKz;j(oYw#M zd3JzvgRD!h` zB>Sz>v-ereQC`ha70vb{X}0Xo97I|i$yzidTReSI-Lao}t+p7owW#g11l=bGeq#*w zX%*FPRS9a1iB5_dV~nL}XOqzP+2GqgSM(|4qlBPseap$7p-U zI7J8ZG@`WEXSrTlw@89Ub}Bknq}XY6KOpADediX8qH9N_Ywu0hflt>_R@X^;*Xezy zPaPs1u(LR`O<~rTg%+uI3%k1x`(gFV)BP6&%5EglZWP0AG~aHF>~5^JFW%~iWWX=X zP|u<$(DpnXMIGTH9Io&UsQ>}@6D<}kWiP#GFQZ{ElW#9eb}w5;FUMLh7iDi94+5~>nL^s_z*%6>`FerdygS>Jy7?0&_Le&w}(iEK0!b;M^tq+D!K zGFwVvL+ngiEa>!r@xy>A<=|Vo1D(K6Y`!EiN;#g2lwD|T7DL0K&4X^xx z;*3BYM>DEM!(ztUtH$>qCYQ!1LPdwF{x9ay&m#y2Y_u{#IRPR1~@8UZl_IGt18hEuao zV}w4Fr=wFeWBp|BMhQ+DH!7xRSEsGLrxo?bWG*LOL!db3Xfq^CGtv?>ysIGDb6UtJw-7G!Qxt>+@_M(VPV5oR;#m&buZFktXKOIo{Q| z?1(vF{+yY}l*MYg)jL_!ta)vp@h0Jk&xRA!=nICd3%wl^bu$x90rT#!Cha@Z-aS)8 z=s0)6i}sImHjne+mGhoi!U zEDLKa7gdgxtfrQAQbQ6?7o@h(AxLVlJ)Qx=&0c62!wGg!gySvcpTw)9ASSnUvV5bek>Gm+=zPO_4-6m{$$bq#3lAb zp!9@z^yJ;?N$k-v7wf5&%4wkOsZh-6+p1GuN`=sDNvdD_w08`cYEl?x{13%{5PnyQN_y9?`+ z3sQ_rQN~NhnEi_EemcJd(zBm5svE3ev2up@iyUj~Tx$(HYfDUDFWF8G9XhWHeXbC) zufmwF%ayNR-d$HmUE5%#tC9cGVEcu;u_HQhO@&NL;s1^Q^vcnA3)+{)pc)s}m{{lQ zrQZ7t70w%Z^_xPcn~Ln4$9ozFv3R4iUk+$DB2_f5ez=Ze#2>_t#8nqn;#PbKVTOf= zBEQ~6uieFh??B}D31at2nD;?n7fWNw)KmeBN#ENkklnxb2cQk`j{kfukxWShBfkzS zmJ3q_eyAaTtP^`|c>CDo_t=v2*!K0Y1M^XxW$Crb4!bW5r5X$#I}{w~Y542Y=-Sgb z_-T^-d0Omw_U-e$-}8tV6y7=XCK!DWHy#F79UABAHVP|&0RTg!2K~AD<#_Go6#Q~d z4!#rvU%v(4_<>K)x>wJhm3=ArPyt{#3>v*jlW#c4c-cr;RIuS;6g2-@OK&g!UQ1Ib zfRJh@|6WVCUal(!(VL7HzM0uohODJex0hy$C*Z{Xtflv-{1{1ITQ1KYXqKpdNyydb zfs-$VtfhCB8!6`i4EnLO2J;n4g|>55mdpTnl(P3f##srIey^n?vUh$o?-v*TEl1N$%M7BX&r$mFq~qv}mK&A|l6b1ENYWd37HTaC zet|21K|yfD_--H8b9;Mo+q9SQ8MvB5H&H{5PAP=*LgB_#Rf~;R&n6M-8;e~#JNGQl zHoj9!mBI~|YD%Zfq;W@teIO29vuxfgp>*et&vu+)gBqT4VNK+l~-kM!F&c-@V2 z{c&6ni8}hc_R(<@NFnORX(5bp1yd~vhzS%gNqkWc!gPR(mZe0W<>&($BpkAlz24ja zrsHrkKQiA)c1am^Gz6eW`>ww(Bu}_AG)N+%G6iOu;4aLJTfwd0gMht zwwAMO;djQ~BNYjl^rLIFR6tMaMu2j)K?+k;lN3=Zp|a7K$kjl#!zm)PWhm+4G1q2R zeq)cv0Bwk@-&mZ;eTPdTIhzirNz2dGmreSpxfkr7WyyjaV_ zdvt0BjvL+|mGK=jRVKcVkE)j2k5`J7^hT`vBN;u6fn4Lpo2t2}h#F%${`UDiCV^p0 z>giODjM`Z^?}TVcZClr=p+(=T%LW;=407g4-luJ6YtHg(#29F72a2IPKFLNXw({=; zYdW;klFEs>-dbg@%_8#j4M4?z$jJ@keJH}7nDGxK%_C$Vph9!iZw5Q0>g;|a z^=N@NpjFAun?(QzzyOoo*CqKS=h7^;GJjazHu}AdY+p+?5m}zkKbwD~NOFOQA%8g2 zukF-kVK-I>P6dcRpiX?)l4y_-*ag)>@a(gK7C1hmTF7pWoiu zEqMvJ)Kcnh9}SE)*bYUf7Vt^P1lHWT8D2&$aF#j;mX$$ZyXJi$$u&0?hb;74zh?GTp|!|~2PCnene(`9E%tn`>h#i=nQ1u3 z+wL*>8UgNKXB&|0B_#6L|6k5FFt+plK=h~;GUaOXzef>r*{uxL7W^}eET4#`YFC(7 zN1+(VXz(w&90a1bF_O)Jn!PXokI3UPXNfpEwqI3J$?k?)1b{-q@*faAZI4D1vB(XT zBJC2#zs@$YerPi3$25~UvX-LdC4@mC?G`Au{65=APoW)TGoJQBx}y7iwy`sU@aQ}G zph~7Y2J`DV(UaFqI|0N|ZamiJx+AZHb-DEU!v#(dqG z!B*J+P%dY)>O=L%mt0h`87T`K*o4UC(llW>=WPmNFt*d?khy#c;`VYIf~{Nl<-=+Mb}A ziXFxC{`bH3m&B$pXjV-MOlh)8aLKpL^0IfO@+E}X1=QY?E}*dtrH{EJf2||qe%D65 zWPdez(i%UBhKEVBT!zS6iPMb|G5k^f+w;d7SyY_LUj{|k!Q3h01bsohVDXSEF-GbJ z2W3m+)%tEs+ZF5;gN0Yv9hg2RjH}>GFe}bGX9Cv<%^w9%P97A#Vc?wcs<+h6(EkfZv1=`YE51QyY!Jw^qWM= z`&I*|!90a88# zBPak+2-wlX|As<$5P(mbeZ?RA4b8Jg;?+|w5_A$?K*)z+28S_(M1*)qpwZ+t_c)65 zZ4YrMLnyD(^h8BSFX;n7^mS%l1la9P5~q^IjPD>XFoQM~gg3=W=e-F8lwe89P4O#q zP(?qRWcv?0>*HdqvJXl#ad0r&>BUKghf`7c9xy#bV8ztghyjt| ztnbc8ovWNtic7Wr#uw3c`QdfaILz<6%nmA3R-% z)8$bqGe$ul?+8A>w&aN8+uT-}%`K}K$a-o$ObOB|Pp;RGx7%?`g6iic* zffh0NWBVT%%BEspPZUseqRC1sXvD3bea4gs5Le5co<9dB(P}Fr%8Nr1E8In=#HDAB zlxUev2!0(ElZe5vubc^!Nkv1bQYApMZU+seY9U5i#fQnzp=J~zgJn9+!*f-hzsgPi zb+!@Gq9RZth~FxbEM{xT9Cfo8-`qoA4<8?6Wt7JL{; zIj9YPFSl*yL|7Zsav3T` zDgt5`o0<#(awoF|+xFOa&Kv}Dm-x-vwt0KbVl(|RREatCT6@Q=OHvG2_D(@Lt7E4$KJEEu7>XTlpL)H6EZ6$!KwrPY zocp8NtQ#}lMX;=@2Y-^@C?x*mdt_ALKDeBte+|q>#kfs zZ_u6>DS?g-OA*d)LFU$HA_whjm9XyFNXFtG%?=x%oZX>l$ylsys%44&j=mw=*@!;j-hEhmA4GNvSsMxk>Lie>>~ry1e(ZnWHGo>DZ+X)|$!joynpfJW|E>ilyXX z5tQ|@`B?2`c!I$4U1#|iHaEEMkotE&)G z!G(*>8$0-faf*Xdj1e&H#l%!q1d}jpawv!f{a)x@{F0`3FrkqAov{4j^!yRr{gKlA z5yJtWCjCEkyOL~pQ0Ds50Rv!61JXA9%x*PE7XvPJu`yjVo3*e{i;0Qd3CZpP$me{( zCh{|!xMOvMgWR}WazTT$fjsU(0yaThdPICSq@;ujY+ta=2|kE7YQUF?VshQsJ? z1T*pAuF;23Y4R$B!z9y$(Ao&g*$|2CYvfr-e>nFCn@Cs80zN?@nhzlA6au`(L-S_s zU?E9IIRIIA@LCum+Bd*ppN5z_j2ahUO*gP?+Plsr)Ub{h#uUkVkWCsa2l&7a5c#eU z-x3ZVK+*{vRi+Z0*CEl7RQh{|6`8`~O1iVFT^#uOGzX1e@++n&5;sqh669%eDI;GPFCS;`Jq7-tBN20EkT#bf>)8dNTMoq%_Txu;bIU`*AvKxmxZS;WTA5F9pW}!~Jl1@&vG} zktI2ymBAD;Oo2+=J+N&|qN$J9S*oJhFs5;$h$`CUw$>JRZIH zGho4jX{vj=7^fgMFABbh9V!1qL1=veh8c-~Lar1_0UD<`avKHDAn~hiN_ApWg=xej z#zHr0xRF#WWfV%;NJ2;sZUU4q%K-RJLG?D0P*=K$q_&9CIpYSlcq_3G1s*1))l=aR z%b7{kr3?T%FZL%c@q54ou}gRjix6s+D0r6q04J7s443FGVEHdA>)G*v6htBfOUM$6 z&Oeq;&J?5Ulup2wA=8%iN|r%emT{*PljoIP_mz1@7T1(vRb{Bw4`Veh3za;mv_7ad z69?2il;cm8H`JFON>wm1RuBeMbQe~P5Lb*dVU4y4^xLV-lnKl+sVtxfEJZ5+c;H`U zQeN}q-^h^J5-gk&teDKG!gQ)~)2!0pu5yK~)}^U7kgV3QsMZgvRxPZS>aCXEu9k$Y z5u>T$m8=o9sNox`!pW;44XS~3$|SJ0yfn4KLp1=?+VO~5th`!*@3pVmt0=qxWJg-m zKdPv_>P}MYn0|z~EYu;Z*9pVbbBxrnf|=_NCF_v_>J>`sY3%EnM(V#x)bh|Y`0vys zm^K_oG%!aQDg9_*du;f0*q|ZQsQ%cXV4$b{xe-ObQK!7Xc%<=jW23lNof)c%sX-I9 zSCf83qaA9UgFT(pqlD8)nuTJchC%aNk7nJG22VxWkDn#{3^Hw_njP(1d>@-gg<3SQ zTNwIUurit>qiR8@X5OexsrF*A_9clUHOXH3AwQbs9$TsL+tQd@jZwu5Kevh1wN-dg zS3S1zV7J$iP&XK~vpKgTTDJK}<-F2|ejC-k3ER=e+(9qV(H|uWSugdHbc`B^e1Ghq zsOy*}A!xuytu@h{YwyI}Xs5vL1c2jXh&?6T9y(W!@Tu-FciJh}nXxv@DH#FPMd`uZ zv^d<_UwA(cLVx6e{WvJ?O4$1>ZS$ISUWC7!J634?yyc{h{AIE3lbY z2K=A$yeo+LD$+ORIH_^_f;|NW14fx``vwKn2V;bXk_?Bkehs8#4ngpFnJYutPeZwX zc1uyCA%zZV#hLX*nZt@B!&MH+X-~s-!XpiaBRBQ^O-i&%nSTox%27tOHpP1kM~&a8 z_AA*93DeegjG~%xTyKpc6f)4xik@n6jIoUM{=5GX1~QKD{U47bh7ys9@^J?xcY2c9 z&Hrs2AwNm08u)LguAXT6fF6Zn1wPIgg~qbEa=m)1o#MvwKmNy8iuhkdLKfNS82%zK zDQ4R&vi;?M>?}QAtL(&nh9*;LsWHt{Yfxz{p({WLGeqNkA-J#E8A;TqC|#n~WjQ-+OD zQhkd&po(ocZXmgvjadNK2Q5gcTy-}YN#3Sss>zNGX9B|?DzzES8{xhg=P;zE77GIn zz>3FvxHFTI2`n)Qe8pG_N}?G;+D{k25Xagqu0B6c!9m9|cZ znd#UR$rVT!PKdp|9Nf+RrM{1)%i?B1D}%oq0r1z$`o5Q+3M~jKfH6&+5L58ypH7Y7 zK~}f*<2Ymt$XpfJR!DSC690mM{%}zC^NqWLB}_7yPE9o1^-Z#g*01k}Ri32^pc0r> z&4d~|EI3=4ESVkV>QRkk@rXW0sW16DA=J|}yu9V_-n5`-Q8gmtYnPKWDRJvMyT4qj?ns6 z^e+?AMhofqhW^3yS6OBl*2vHhEEN92^qcA5b{RhJ{)NR_enfxyGNq+&lm{+<&@iJi zwTp&?J&35u6#FV|P^pMBb!hYt(|-k>FCW76*T2eGbWmZc9~yI9w)p&0iI~4ve%$#t z)BhNWyFYzAx^jlH)`7lrk#WLv`HK9kgMRq36g&|97eSlgAgfxh@cVqL)+cn~Kl4E+l58QP7qg3VWf>6LUIH zuvVDKSiUZ5a{|s`OJ*x9Di`y^W7`k{XLB8GUbExMsmCx;=Dq}#M0bx9q@g5264tn5 z4I7`t*+FO;cr^O2;*h>B0{}176+eLX#(lU0QBNC{AAFciPsJC6OfZ}&C+&=f6l!aG zzE^vHK|<*u?g@zYr;t*_FvZklH*mIsBW7PVR}+6zpE1W3SDI+eZ%&6a8se1m7@gfW zK#7eKjA^$nk=bXVA6zl-?rrnVexopS3CtO9^9l+ENT;|YRPwE2cdOToTih@p+JQAf z0H}5dheynfPy@hDtaSJc*!pGJ2+Q$P+$ObckvAnmWuv!62J{UTw8dr=7y$3i1nROe zVWQRy&5P!afb+g#W4=^s;GdN+`i@UEZ?|E)VNw1yeCSh2lrb*OB*8K|4Jnkr1fk%E zKupFURQtSFPk;m{;!$ZOX@>B;+@7DzO9~Vob(!KO+Mt7%0>fF6K}K9*X=Rb^cB}wL z6F!>ONP$Hvj-R!**#q$uno{i&GGn2Xf!mfvY(r|XbxFlaJ4^=m!+PU&Db1rhuRl%$ zOc3f*2Uw&i+?-V`crh~+ZFafa@4;gqV(T;al=cLFr$~6^KVOYnw;kMB`t52W5Il~G z-?s9Hie7)fuH0W|Z=1x2b}Xo-w!IQwE=i76J1}#zRe5QlNavkAu&LX2*l_;zSgD(T z0pz3*@le0iOD0pEHV>oFORCOJ_PJrfA+4^{((Ii)v>G~+-wPc?f_CfAZlttC6)>jf zy>XUUv9%RqUzi_nYHA)karAopv5wK)GN63w{P}TZU!}QiVeHiHEOhb|V%Ge;uVp#x zVrLgaLTk_eepRb|4GOcR8&TyvkmP9{NwuYiVEjB(=xGBG*V0F?auI3pv`IM8($6=3 z5$pA|MS_KE$=J?D;alr?2jP(uJNR=o4*w%%Y)T_%<(|29uGnP7x+1EGyUT z{#~;00u(2_jm7CP3YB_6(a9pgaD+9kT3DCc>|lwv=`l9?<$B<$S+a*fRJBwzF*bF` zT;lU$V27wMCdJ-~#~dL(*vJ^Um1p*>N=kdFY1UfuMr_`9Wc6`O!#(_oFYQ{N`{uLY*wuT_mqha| zjM*2Y35J*CR`R`~_C@5sea_IYiSq?E74*p&a!P<)d z=j9NDhbeif*Wz5*m-Me{Om&&-Uc)rrkCoMKou6?RIpo&)pN1t!ruVOE%x-lUuZT0m zt}XJFVp9dgu!MJ`^j9^uWz|8ON+}xpoOW9An}_+;Uno>MoHf5Y2YknZcFXon{I#qR zHtCb9vRZ&4jQ!8wh9y5@W0@}ghpnB75>3c*$aP0QnA--<=N+g?e7Onn66Ef|8Z%w- zyBd2sKG+;gWPUr|(R#HvS)iDs(%JTVIRxQhcDCQ1t@lQO$yK{L?jgaW|5}aRT^;?l zar=eEx;Au+y?U}<4|oEyg97zA!q*|B+8q-~94jqTIRY7oVM&oAEl-@x*CsI>5I?vP z!60g}DF&!uNQ`3aXE2wBM#0SuibbcLR70avD~KgnZJ7*<{iR186+`JlBg2m0Kocd+ z8v#n<(pj2EgKs2UPheGvD3n!w%|Drf@G6}$`HgXKkql3$?apT>QvoZW0()T^5XwQz zG&E%AU{@^M6=^T`TP3mvj68B5ro>y?!}u_CXwZb1G?vz$SSwbUt%n~rE_5RP4syI8 zDzt}!Tr!=7O~&V@GGi;=7jXRqF#qLBy8_H^r5@V&x`6-~x}H&5-Z#GoB?;h9>9%20 z9zFy%ySbzSzrMyj7H^DgvUyvMfAmfh<%JQ`rUqSVL8@CB_nqm0(PuSd8rS8?tY#nk z_tyFtE@i-L(KoCzS%;CJ_5fp&D)IB5<>$VLAKzF3p`K97B=?xM9p7BOuDHwNm@{TAH(31Z{^JJY)nZtq}WE=qTL_s>kPz>d+|s=saA$zb${{Sg7> zwtd9{l?V59ztxAsZLDYax|)U4>_guTvbK}SgYn0+o)Gk>-@&5}tu8KajmrVPGyJ*n zk0d^8b*kXqmX~ygPIvD2HW|JbY5+`I?}Q4$1VAJ)DP$Kat*n1OH@t z|51(63I1=@*dN1EOE01Gg)${e9KqMLd?5!FqA4nBZAZBlhZ1$JM)xf}Qn?PZ3Pkwi z$KDH3%HNsZD2JzlUoeb=62zCF@IRSeIYyM~csm*z8hZJERbvM&?>}5d?cC82NHQug zCL~Z6vwXo48NwGSWpt|8*>fyaq^he+?M5qN50f7?Bfd(@zJUK{ zH5P5f?ZG%kt})}DFuKhXkUD1PdG&ey3G;Q&U)5M6DV5;DkjkBKUDkPYu`qhWn8!_Z z_6;eS=+!;A-RJV^Pc??9Fd0aEor|PgBC)nP5$s}~7fV7f)d4-#3)~{dt}Kz^TTTsi zzAm766(viQFa^@DDx^~`RnQ%tPVq$`WN9P*rVpK3_f<28{-|y$1Tq>qR;gj|DEamNlfjAl9!?NTB|nndACsldo{a(!-pVVu_-qj{?irjr z&9|M??mwActEZq3VMhvcfCY+ycVuK2?jS%gg_C!9m5V9Nf+=S+Nn;6cI`4l+# z^%yji@64cS7hN6(L-fx4C^a%5RNR&bFiHkiB6Ree(lj`c{AoS3=8lqP4t-Z$1&~CX z#1|YXP#t9DZj6o|`W5c}OpIa?RI4K8RXrPzfO3V|HrPzDC#d{|p9Q3fw@nC_9y}zd zMxp2=X98|RjNr&54B%&tK}Vh+2h^FXh(eJ7*Ey{2ZdD{F^(Zj>p@AUr*uZFU3Id8* z)_u#ipx`e9a;2Oak)mZ&tAeP%q=?vU@JZZAG4u2fs15B9)^jI{*rH)!ZvRl2Ze4*&^%#+m_>$&f8) z9aR@L$4)$dJpQ~JZ|?e=hxu>SSSoDw7Vb}tQ0nmwBuF(D#)s=ES^WKHR9_#pG0ZA4 zCUJes)?1h0NYJk9*8kaR%mlN2aX{s!?(^?ztbKW5{HCe>?`lltw(ZCBsry9x&)f0a zuE*yyf6R_`MAf?|Y7KwK`tU2s|AXLl?txea^d#)W*4{Ufb2WFPJyAZ|F3cCPK(;3$AZcE0 z(UAME<|iiFwHp+V7a&jWV}?V!7)HX2B$fH0HOcg5c)RG=>mTn1!N`~^BtWjJb?=ZNE*C_{9;Z^Hk2+(j%VUwQD0m1jG&+i53Pp=8MZ&t6v@#os~pVVbS zA`HOQ3&3{|AS_V;rv`j#ruo(sU~278${;{-Cr=g5PcO&9c&AFsANaaNp0$OW`7V%K zFQ^(fh&L@ruq8;?#tkPhNQ5w09WmHkHdtcOO^Yi?w#DtPH-H+4SD_`?x!FHA?45>t z2q|HRe{qQ3Vu&&SM|9H4E#Ff`#J>hsm3Spe9$2HnT2o3+{w7^%3c*%y%X8S4TL25c4+6H^@ z;mMEUxTNYTNSJ&q92L+|`*417g=bf}+U}0i<#CJoqrtCWFrqyT!f9#HIJ_vQcSsWN z38PlDLOFSx!|68||A^>Ow>KeZZ*;Yz$i6>Q@-fn2S)lP~fhM^pTiJl7X8HGo?H|_6 zxHldx7hC`DYC6b#1A$@P9}eGUJphTWWIYg#7h=r>KsYV`E&G6IpKLTYxQ@OLbO>5$hI;v=x zw>zqA*)2P&>bN;Ps_sT$KCbD*w?D2Oq$~f;X*oKspOj%fX_(cqKWSXFDnDsjaXUI` zUJqqHZP`w-KW*JFDnD&IZaO;s-#9JhXI)P>M`vH55LwQ<;RqaR%4ukndWslwYi$1w z|E~{450wKw1UW$i|F#Z5L>NFHg#ReQ`1jkhKe&CGG?OnQseiW@J>ImBOMe8e|3U`I zCH}|u;-ACbKkda#nbF_v#c-Bl^IyZ>)xX*cSk&LcUdZhk4zuxSOYPPGh+6Tl+p{#T zzuJq5T*)XhBRm+}q9WDbw`Y(?vBrGlFM;d7KZ@ik9sf6vqRK-Xc>|nO?@lSi<2QHNd4-!y#m(P6~{>P)}o=XW=Wd+f#>DW;EW5rtKhLF0Y zQ3MltLa??p3E`&U88Lbgczkwvz9mT=19iyjE`~SJT)TrNF}#9B`LW`!2hHOmp9#0) zkuc;7O=YMK3KJADLJJaA0tqcmn@w=WQw-e>{^lB(8r!A`UKIZ8Q6wr(5AHPI%W~f> z-OKjAIoQkDo@89i3C6eE&vQdTSrj~?INUD?i!88tyUD)HP?%sX~Q8 zKh@E((D3W_l6E3@b+GrMpB;Rt>Q!LD z|6XZqSIf7WGsn#FY|VE=yVB)sK0NE; z`QiPse=`_+f(#6rzUnVP5pmI#CRnayBluKsy|BuDj0(I9)==ovJyPV!S!EinS$G?S zuFMJ_FNv{iOVYucv*`kUb-T7ov+elPj#}w*&HvLC0jdHK}VmxT(+Z#C_6os9j*IjpwCvS%(jK}PJ)ioCq ziaYYfJ4;M-Y@2R6%PKwU=lX^Ief+lc=~7WwjO%)e3%<6%)O^Cbzs`B>Sp&+Rxa+Pb z@oU-r9=&$;4Rke|=s4ycX6f98fwWlOje+si3w=fAv$^$iyTXGMUiZ#M+*?6K6P+%^ z(j(;afFD=zt28CzFm+=p_!X}Z40w1kR6giXHW*$tgcfJLn+(>$8mJ)D7C4S!H<>I8 z!7wG{dc#o`P}H864JM|=cr}eqC@f4?&XU~l*jlYB{S<}~pVqe(G92WkkcX1vUQM9G z;oLG!wb?_*K4Ci<_4b$o{~50D>+@--&ks}L=DKceo++0Vy^XBR@P2_C_Glhl%KN*0 z+3Hy*Ewkt!P!o5UhPc@F_7(+viwmN^=|Q1r)F=MB=#0}R32EGggU5t}U$KHC^Uy{p z?~@0$tfB(r6!MstB&xM5&E39!-Z9vh77H^%ac#>)?bJD-SA@sW?!Cq_g>lN z%dq?C>q3U#?S&^UT;25-_Lx$ozlXj2w3)J>V7`3*DM_KTDKG4-Fk7+SOR>RFrja^4 zTO|0aR4t}Vt5{*KO#c_8y*M;V`tZdc>R)%k$PgRMIdt`ZC=*O2crgC2?t=fkn+=D4 zmkH#y-I@Og^P$)Vt%|!d#jX7*lyCvUj_oqrUkB%$a8ja_< zRNJ#p+_|Iz^3LS7iZyvp*6e&9XIJCN&$r&1q6Sm3XU~id78GQ;oZoyMy+^`cV93nJ z4xaoH1tuUd-n_Zkx7M8>VE1~sJy&&$`?HAqd-U`FtW3DWhGS`hFlX5NAueJ_nXnbZ zzgzN$Ia3hBw0XS+lEVK}CZunRUBQY__4CNT2z zLR&Y}f0qeKwq;AxpCOVW`Z%ZIR*hJt{zdC3*j2{;T*1v<>qO63sr~%uu5{~shSj^h z*hF4C#)M>=+-N;lr}(-Q_Qp zrBqDzvJE$PdzEeYN0t>|eUlDr#EFk8atG;9Su!VdP^n9ru^-JErX9CS8`n|VS(8@W zK4WG08+*JfKMFl^Y+E+AwTaou@S1AB+tsh`#M?P~U-Ebu`~dsM#_3ois7MwjbG*<+7IIBj4qaPKpEPFpGT|rwr?W;o->bu*1iA z?gj6w@t)lkwh3{9LoR6`0qemeCgfX@Bd6GKJ@bRth+|ewQG{ns(clCyuWdgDSWxwP z=2=vT>f~6G`CJKDwk?kV{E%*c=3NmV@d2#5{xAak6g~oX@~!bbDg)L7P(|4`SV?^O zHyN18*tSrFFg!LRRmbbLQ*xp`cB$Tv3k+oYIXdm_!%bQhPYK zr|a$&ksf>zIJtvS>!WFA$n)Cm_!gAin|00lXq0N ztLEf3xEG#$R`;y42a?!^!yFF&7moY@2abdf{ublraR^`YnT@oftXgSb6b zjsJuo1x(DBZurBp01R(DIBJG6WQ}tT{0@@>&XRA^PBlS64-`meG(A}JrNMMANnnk^ zZoInlfY;u6aOMoXDAP5egdLO^d?mfFJZr*4(I^4nAh{k&?2BL-;d~VIVkr{N+6Vz} zDqIhS?~E@skvh0k`1N{ytj<^w#!s8r@bogQ3ANEi!qmjyO8R+Xr`|hSP?K)m$#hQD z#=5l^d{!$NAit=M6G5e>Xuci*q9}txD>Nu^Jh-JeH58(}X(|&Kyi_dXjv|%fiGS3C(Y z|G~L6$5Q9(66H!e z-I3+_wwr(A+_wJ%&P{P;llZn}P`S!Eb!25%@V0d_rpl%G{}Y_s7-w-WMG+DvKekRV z@~pk#9pd}-Va66FddelOH=)~nehKxJ=pCj8Y$M;~?VsLzvC;~2Mi9gQH&{m@1*v?~7)jp!O=9YTNquy%a<9d#FcHR!#5 z9W+RLO48Zc>D9D@iCr9i-=Xk`s=5;eR=t0UxD$Ybm&)mctV48c-85YQ7q@Xv2qBuF zk07UuI1{yPx-=&!!Xy$PJ~o7CI!;s=4IG!Z$_qjX2wRV0uew?L04rZKFZl^*K@<#I zy9csEtZYn;0u$1i#&FP;7OHSz$V~e6EfLOwBqWGvV&aLCs1sge3xHA;Lm92KP4wX` z6Vb;8B-3pL8wfJY`x*B#A>YJ*?^pa0nD%r3^ef?XWsq+oFQi`q$uRwColB)yj3KS_ zQdU2tb>7ulm&FK%w9eYek&xE8=x;rkc4?VeQg=x{L=P5_4Z$E*60Hv7RJG$ry#k>P zodasJvqRI(bAY)4#mJc{P+qaW4gs}baW#tC6<@FN8aUwkNhPt8P-W}#%>%2#GWK0u z%?h?wb?x1-g+wWOlKojHLX5(xbaPv~+BfL9vTyAq=qy;>_{hf}n(uW+6Y5ZgYTmc* z;=^@^qq_1>T7x*k0fWd9-hirGI+H4UvnJii;yfif(0=Q|sN`3#=CFY8K*)ygvp9elhW4O&*pNF_$1{w<$#`V)q*90( zOhnBA=acsp&;rwfM;huFd@?Dbhe(!yjPMIbeQ}al%f*&~94=%HA39r&A3v6b3YgEQU`zr1aa)x5o@rX&W#IV$vO{bk&;8Tr>zVeMS#JM*1;r6X+E?;h2niebQZ^dwd@0D z%?KpJ9?%2flLdcx?I36S0F(;aFNhH&G6I?`vWMWF3DD3`!&sWg1m{A?OdEqI*G27F z9_o~ROL^mHW}H#mF|x9kNq4UcoKRUkqDQrZw=ql?3%UWndUiD|% zdt-5uY^0+hu<9CVLeMjs&D-r(RmgFQ$OY9tkU&3W3OXqdAgx5bL^KsihG2+4W@C>V z*{5eW%huVjBP)xd0prwQMn-GvrV=TYQr5tfQW36*o-i%-MUjTfr_{hXLp=KsmsPY= z3sBz=%6&;SI=U(g8%p zI4gqUcG0+cjQyqkJ52Q_KPqFRi}~pK7}8(MjJQ z)N3=vQ6TJdn+s>_D?~2NB>awRTLH^K6&TQ3emEJo!Y1)M$#Pv}lqsi=1(%>;=C$7&Q-5B^`J;wFp zx%7@Za79!JlM5wrbIPOkvi#pn8oj;_+Y7VdWNu^fy&2AReX=Q+YGY268UDf?15$oZ zJF~pQtu0)Mr$!PooB706R%>>cUm-YoMS50VIemE6WI12l%cE^4f@^eDC;q{(qWVjZ zeIA;S^+1zeU5q_QYE_w=D|!Ckm0WhTBbG-uqs4-vq8J?Cm70`%=>v zr}XxW-Ml7`VF6#~tb)&b%(otLhE;EpM-2AIdD}YURj|cpi1uZzA0{ZtTuRqy)}?^_ zDm5x5RjATOy3G%>$nZB0k*jM8Yj;b%**Bo-)#G<@_dhl;etnt9qB91|FWD!2dbsJN z_ZoOyJy7{of@w_Ww9+3Pt zBd)9a%f$V`o4Wf<7g4&(3CH92m!8wrzO?nt@+UlgUh`o;7dr=@&l1U=R4~?-zpKih z+Shrdvb=4X$dSKN&+$%_dP_HR>+`G9@9F*foX66zC-AMT^K+WT+q*4#zuSTA7qYON z%M-TEhbc0@+IgdsU*wlh$k|}QF7m7A0UI#AC$yZ)D=l|7T|b00eTxMr=n@@RB=6@> zHdu1rm`DM)du{}J+C&VWNCpFt!u`mR>|fbErx$Pw zPjOcvxCq8OH$TwyE!)wcJcou2fPxEvs=AA;C~?SX3BxfZDkY?cxRGqr!s_MJ8~1go z0MQ6Qn7*_qwz?>T#i%X@EXO-cU&|;;K#TxgOx8Y`&3?czp*`V-Zd-Wh&%2oQ66kgQ z*loSoH9frb{fJ?HO$G|()BqShi0CLV48Ar>A1M?L2&11C1@kTDh9A%`7mawqZ5t`w zLZWj_vu$WcBu*>=z^Ff>xH%F4tY%K%O`5hL!0Qo*AV5KOLq*N_4kaCGI2Ug51IjMM zFa*Fr_`pk6gYP7+!0GxdSq3&2#@XGsK^ z;cdotj38_+8u}{u&xSS1SMP%$gp!X!rwx?SrMqrr2pY(?NX@oD&GFgJ#H>;)_SLEA zGAyl1T!7?=Zs)`x=CU*8>b2#(@icB%&Sh%L=@!xKvNbJT$qk^-BWKdUw#f};d^&2t zH`x_F9GmBIlIK9rIo%aDkD1>QmET#FKmRuW(|-OtkNkC*GRdY$fw@+}4kpQIM8VmP z+L1^E@TB00V*G@x8$)&hiLM(Yo9D4r;geKu1d1Xg(IOPXA~e4ujGQ8@?joGkBD|X- z00k6zA~BMo(7dUTMJGQCQ-jP-hN2qOX9?AugJPaRl^KdnrHe5viw}mOi^IjdgW5hR zy6oK`L3Rortn6e@U4Uzee;5~rbTPF|7T!h)4^jzcXeox9c7{BV&x%dMo1A6 z1>goTfSi_#8cN}YO8bPC`xH|!$a;{ZRy^4RcEi$iVCzO1n-yW3+RmPUWDx*AGaCQ_ z_1>jSpbVH!;Sg%BYjyx|xuLmeD__~j7CTTvC6LWPLTzddsW_mCEMvljl20>~C8f&a zK&t^MRlcW&ZPjIAj0))nK$cB;PJX3`=JL1Q)SRg0kTd)U3Z+#GjQ$*I`UN0}LN(5V zwRrV;a2$cuV0CVDT|!trR%rd)gzlHp`X@i@Im&>ier%U^d}ndgtT)8h6M5Sz>aZN1 z+k>avD(a{oDen)6k>hzVCn>Q%664x);`eBw->MOd<(%Y{r1>??K$>cRO~!mpZ;_g- zyP7a48{f$__n0@w(l;9oG-y+&06WHe;hUb^4aq8?9P3%`+(NO$=>)*}x35 z_Jx% zrP16&wLQMMQ*;hazGBw^x~OY_(PAuyN?jrY?XR^uh3Pxzu2~DQ+Y)kFiPsa9&ARu! zfX*B&bz*>`c*5dI%2I#A#-3+nd_BpUEY(I`(WzJ{O+7+$Ju9)zk>iNxaT@}IA3fPIhn`d(@EjfeNeLHqaz`m8qk zKKHPmQWBo27OdU&UEOx4+5>kugpYH{uCWPk{hg`!`a_@f$GGf> zsO7--Y>M=G9)Vsju`1`Fx_G;E9`I#?wyheGRi3p$!idq)uzdp1ajMW+oX9ng*cxZZ zpOYvsfjC%gIMm_Q3r?UfEP=<2iqry!Czc8r@4%UeLz=wimVyIJGv-XcBgu3i2oE5R zxXBy3?9YUuW&1UdoM&ISJo^leLi5JoZyo*VFCp$WWy)jdV^v(uxmaget z`tm#Cgm;{46MMCz2UL?tbpVuUcCFs?b~>9^~g{p>GTIX?lpxxzMj-$%^9r#g8rO18j9J-k6DSOmW<5Wb!w&IgdbW+!(fk;APNnV!|*datVfUR{0s zX+EJ=U6N@2qWJ5@!1+a&`^9al4;XwODz!gYxqhJFTk4Hju&-MX4qTMsTnLw0cHdt@ zM_jI8Shhr5Av0SZ7haSwUaXy3U~^oOOa9SOH=(241>4UN3>gwx{ZZAIX z=GTAB@4b5QDYW@B!Q5x`mE|_aPp8vt*L7_B--r=3bShHUue!gi#IpmO*swnnFwP*%>xVh1Ay`~F|i37=?pQKQ#DcFW{~z*+>=dX z;8zI$S0m`xEjyMr0RjmLHmQ1InVFK=?5*cO>{0(u8H1P>GKk7^(0N|K3_&vKO}26-LpO^V>5qGYb5Kez zMC{pA6uu6;BE*GTfM6uKwB_OD3l?!YMdl60>WwO9*K9MST4NWiMCI;JkF?e#0C90F z3wWlRo^?v|K+iaqTV=tos{qBXA^|rnhj>AcK9HfOe4pF^?9SAzBDKWZ0XdPFho!Tt zorrJoCZvC6GHx(JQV?jhQN$RlfIPG(K`eqf6_Jy4)U@;(I-rUHJ^#~V4Jg+URB)r3 zB5{k#0VZ3A!gm}H2K|mlP_0E;QgV7@&o1;P3q$KiB_1oxF12t1IK1&wmx{fU=HQJX zfE8bkVD8xEqi&cq7=~FoTX6P*hgI#z3sCjf`tQVMg)em9u~mEL?Ga`_B+1@O^7E4|VlxRN-zq-4ED zX&0ekmxCZ(kB~U#B^MSo5J=PBluGWG1>aP@yg7v2D9Uj^jQ`7F!-;%e=zQC$$1_rP zTef$*_BNuQhvQru-gmvrZ7vw2<)FX0nS6KGaeX(=cHiBMZ=i60HHhA~ZLl1lEZFts zo+b433;qKO@WmF4ZPxQ{H<+#FWzHdtl}hm8$IFMGg*>M(VX=`glpe9K^RNee0OlVK z8$1t&Nz84zJi8(LU-DqT9X5K&2S$U_FK^R8s5zD^^V_Ofd{+O(Vf!-=wx?C9>|Txi zmrlm-1l60rd1nbi85gxma=yDvLG49 zx3&O7@U`%)oM`F-C?`b23tx_p1cgZxOg^|N=}TZ)t`mvIzOSRB8%t!Wxel7mLfK#g z7ZkFeu_!ug*x6a>C}35q$+7?HQ2D}c-c=)2p_!HvZ`jROR$SzyVq!nf=`W$!V@ywN z=^n0Wm^x(&mr56cuQuU6!DfK3HrdZc&vjSSw68l2J9ciVh&grbUD$nV-fy>O49E63 z{NAfIq59+X)y|)*O}Qv$NXY08yn#nFwT)hmTt|(*YEL*_{V<>YO@b=Z;dGp}qV`vU z%3-C1dzAC%l)#e1BSoMj%Fk(WT2FX~>YVdT?+cz|s~$0Jorius_n*gudM`d_=>Fq) z@O;Vi_wnEhgZWqROi0ck3=c2VBCwUH)}l9bW>=qYy+>@Vh0I z6s#`wzerGZe7`wt9Ob?}YF%>bS-1J3;r6}TIUzItb zRphxpThMd9KVP!YkupfqeH_XAM}lgIV)YoFpfbD0 z8i22>&qj-S%;3YKuZHOOqhpjf<(#};$wh#Gu?)Vy6I4hi3~CO=(zTVtV3(74)6!y+ zPqJNnIa>DQ!e6C-1)C~Q5`7sFsZ_(mL_eI+dU1ZG3jd0jCd?e~zZQ=sM@K%f--L*} z9D{+Pt?&eGo>!VOh7m>|rHB^oP0okV${CwQh*OEdt_)IW9#(FKU@?xdm$BRH7C%R0 zNv%)#*YV&~8MpWNq{+?cU)RSuX;<_hP|ytMx8yw! zef~^e34X&oWm=@hBEzCG2Rm;mCYy$@t53`&`N7xKajTRRRpwLS$Af?DlN{u6HMX*m z23Dy_5`<;bl$8rn$5EoXg}LtNFiXhmaR#YCpalP??=1g|W%U0nLG^#-ou#9GLbYKp zwMQM3L!HTXOuh1G>U#UKk+hsSR=wD!D~b|@`awkYbliUBLykQgzpLaBa4e<@?EbS?deK=|v2rpK2ru9#)@QW-9ZM z4S=#IsN~zn&R@&p?D3y5>&3+ylZ+Clkf!9OjAwJc`(-5*9_;dUDzG!V%2o<1B)N+3qu?^kttD^+ zrl-oRRYP_~d1D?oh)s6clgPeqwjIQ&z>gEJ6qk3=WaJ~-p{Mq-aTVcpI=`GQ-FOulAD7A8NPCZ&Jw6WeP zT{pU%HPVPi3tuERGDbB=)dx7{`$;nMw0?!4QhtS?fMYNR@su~rO9`YCfJkf_U*u2x zd@Any$obVo9CB`RLM)EY+7e#bfqSU%xA+Idty?SJF);mVKw15J1FH0UJ8&9Tl;i^b zXkRyLg(=0`kn_11_5{2tsk*hMA+M@-xxvQtky^nd80E1_6ych-CMu(oh|Etno$Kv9 z)#$w)@qu3A2_wnwhhPyLvg{Lg%%cM4e7i;fH{4dwjqV{GtGM#h7c20)cu8)$Vf^Lr~ zD_;D^|7`&JFJeVVB5q-r<4&Of4;3UvrGXd(1w{-45+n{X5Ux$4^6h|Wmv9zMyP%wu zs!F~hvyu!Rak46>gPDoOtZ-b`tQsGLUfXR9NWo-l5@M8(2kSZ=M0Zk-sHWWX@EK!f zW5B2Ca77bU($NLV#S&)=Lv%eHw!gQ-0IsZM*L z#FB}la$H@ZF|7av6AMPKJRHgo5#JA))`*=zYHm&+U75qkI#tnMl}m>g)uL6TRE^k^ zQB1V*{2`4$Ke5Z0fgKEjedB%t9L#(~ivzxt}>{g@-Wv zTvOBhcf?sP=BsA*n_BY6{aM6WMBYi}$QPWi0VLFmVMT#EKgAzF4Djvm_ReLSm?99>inkDdW(p+W&2o_Z@r7o*f&{APet)D67 z#7$GJ%czRm!vPLkz~BEjoQP`mz9@X8A( z$iWd=@mf4}dK_b8_V2J7Ua4mjmBV>UZ73g$!@5`{`>&_$UW85zSW2aGG9IL0p1IeAOt5uALq z=)c>Q{5bPg(g_g}INQAv&pilms{>C9n$um{nm%d%XraW!DXAASWcKPkf*HKH5^y{X zAVG?bz>$6hvyp3nt6=HC{g2OKI?!NY8HpE{S9Hc?Om&*vvVc!FbW^|;8Os?^4*tmw z7vfEPz{6=3;PsxCi0FJ_@mbm9vVD!0SPLaHHr2cK2fA-OK9oeB*Ws5Pn(VqS+oN7I z{l`-*R5r2C*HzvRWE~?-iRkhFnL5k zJ}a`nDcp-k(PPh-*hAX)f2s>(Fe84(pg-i=f{oN9HfN$D{wA=`M5&$OTmtmQS=+M{ zque)KE>XRELykW)V8NUD9DD9MLxPoX2j+E}YX2TKIAwVXHZH#k9Ua4r)4RvB+d6k0 zWMF$E@Q`*=l>i%4E6_!TA-4rHeMSd`p9L$;yP0pw(;~XT9gwaLe=nI3MI@?=LH{*A z5TOzwJgkyX2Ixj&l0eZ%_zIs64vIFoF+ zj#W5Gc(`g=IPp-p>~{DKQiKFk#1LuNplpP&Rm5I#1mD|;-OUJ2)W|u8NLKmCS@TGS z$jG+h$h|qK5e1vckw^mwaOy*Z#k{>)C59x+L^p&A{g zPgMXoT@iwnSo8NJYhJ}D$3*nH$HCH~CA8wgH{*VS;sa&kOE+S!&Eo^Y;=c{Y_l?H& ze2qUqNx)}K=#)*^wM_6VPS_ewaNA7SKuJ7_PUv7v{4AH~5|;R}G|_P|ad|7z0V!z_ zlw>cHG;f)dZk^Z~o-|#W^lCF{0wvjsA$dqH+0s0@FCy8ZIJtc|*?cp(5e00<0Irh* zo0@|wBfu|u7Xo2N!c zr0Nx?MhvIwZl;E!q&;Uy3zCDscrs7(k4Sq~oaQr}roEZwi2~7LfOyD3G|eGy5s;_F z5Z7Ud#wNrGC0(5%-BB)G%{<*EB3-pO-C{UhWi#CjB_nGzsf97aR4zj)EaQ1;My_>6 z`A~-DR)zvY=2N-M0`$y$i%gY>Ou4~K$&R@D?M!GzW*%~u6?4{&e3ndD7XNUT^kx

gCZJ4%pMFTy8c)z8R%Yd5uYnm zCA5ea3f&!q@@_zt5sP`~i|1sDdCZEHLW{Yf#j}IO+#AJ;h$UR~C2cY#@u4M0LnU#c z#RNC3YbD`T{q#b1L^7)qQWFGh$Iz9|C~-I=tcxg#!90(~NQa=t${W`dlT9@r3PN;* zXHI#L;-5gn)NhC-Rte}eV{s8H%IGVqvmDnhY-~j1P3IA*XQet(JnXw2JSiL;F~|su z<3n$XdCfzNV4L>ZIpw4qc7)AHK}yV|ylfTCp#VgWpPS&M)iTh27$bsO$YvRXFh{X+ zx?>LE8cXHlc@TePAnIJm`XwO3H57NQI@i|$1dS=yXGDXBVD0&-%yG`^WkQ&UCg{qE zQg~12{KN($dRhH}bLF@>-fU(v+oK^Wvur^dHOqz|V;@q6h!9l7I@h#JO!9hsM0$o+ z7S1%#stsALAt2>ZFsfE?1sn6|5|Ew6W8tQT1CfuS2vJ6nmR3LOm3deNdOb=?Be!`3 zPHLrAQ?>3!qbx%>*(3qcp-vHL3awEJeQpZl#}pZY#$C&%Bg1B5#1d8`P4-*-tI_7G zdHUpXV2FT!g2&J1A;K1sc%TG_y7+C2oKdTSf2&e%t4dERrBJKdZL20_+b^VV<>`BZ zRwa&Lv^h3?|JE^?HYAL8bFp@1`+7?b6YJb|*>UOljN+-$=w;Hcz(GIVVG8&W} zzWyBn*y4e=9nmP-VPc&TMx9aqoiVwcaXp<0A3Kw7JHeD)sbXCaqpl1%U76dJ)6W=BmDI`r9H9TgzgxfwSEnEXrj)nfu8z1rBU0(wt}ZcO@-JP^>i!q_cO5aVL0LQov{# z02EB6IIVmi34<-Lmbor#Nm26-tF+&sE((;$AT9->U2O-j4mg7??&c$#8Nv@pg3fF! zdtW^+twGv&1dxR3v=QqpLt1yz8Xk}%rDo3AzqP{FOSUm@iNpY;Fr^%OY5UjRhw|a6 z6Gt6WZUWqGN4g}^(hn|jN0Jvf9v`}i&oOePZfS-c<@VY+Tr%^fIW#?CcNdigC}j#- z{Sl%8fHHJMd-6eK>pcDB0ufcPM|fgXibtzT060F1qKYsU=cK=Mgclt{dJGL}JpcS& z$X>J6D?_J29Wcsyrsf0^7#c>7jXIE4IXvT$ZQU`}c7c(JYtPDBzkXkdLj44g*m|nD z`V2OEmpqQaWJH*y`#9cH_R5aS+x&;bR@;PgE*zJ4C50_qDi6+Z`Ivh@rO@k5d;R2U z(}Ya}xXP7}m9kKCddB}X>+;N7mLz_mYlc3s&tRvNF=KDAJt>`|E=d-0SdMLdOWRCj3RrOS2Oqpl203i&j5(;Yq4rMnmahSBdZ6|c^RvC;J;{u?IM z-T{j!uh5>gg*fKo)R;m&V2;UT)tnMrKa~`YMe+MG(p+{D3TW_p1>kJiow33dkw|cq z+Li{OYD@uXIt2{EVPsSuPgQG9%|aPxdNe-H?0K4|wgO^Rj3w62No6gDn{FxX6S|vN zru7L{G*-&0X{F5><-JVY(%GVMWOFjC01T|KE>-e#ent?Tbaa$~n#DnLc9;&*_5?&q z#AFY7ws-CkHBp!Y#rxd#0Da-nE}6s#xZ|!=0arw3{@a;fBwa=HW;<*edHY!*;pt`> z`0Qzj_L%RzfTlDqg%lkZ1nS|z3U67?B0cdaRsyjq&_nv;2=Xwd z6|KQu1gVC|a^iwjUsgl2kZj5Y-h$O(;qUVdZ7Lr2tDdrjjw9Tp)%09GK zX)Lnol-#_p(oV0`eQl@BRQ(+Hm8X{c{Y(7~cO52b_B4yYrv|=RA8y*(p4bc8U+)o1R&>E^2f_>gbH|*+#NQke+mytJn%(_b8g`ic8xJe@TaQQZZVZCEAn``JvCIz zWL5P|5$rqVL|?V!B8{4GZ)(nTMtap1<2z+P$)A&{z0;ZJ3!665;LFSA+!c|$R&Z`_ z;aaTSEp8>}_Wk~I@ru)?;O?3$l)IG~v(84&7c{rzc$O%FWcytuhZ|E4&cyo8d4vtW z&n!qqeteJbNrAGET<=kd4Y3W&x&c+1w3$4TQPZdaeQ0hsRnP$Zw$3qx;MVz(agEaw z18@5ef~!d^da0F{uIrdepP)E^(hsu|TlLYWi1vgaao-M%poYR9#%z2pW(!dF_K}SCcP{kKU zgasFT3Y3wg>mhLvVn3?jn)#U+SHH+$ zR$xTahrJxw;BY%X6-FiPl)67)6hW+y4ugq8<^r({U#4iz_d8^~3t-Vx`N=Q-=BvCY z>}34S16sG_17>M3PE+vPdvAJ}w>je8^Elm*+e=!wGS4s^vd|KOCu|J75(fx}BEJe9 zd=^UPL!DF-N~vUVZxO7|7xH;Klvas~`O1d?=uDguhBp|d5hVx`7U1o)MN+WnDo*eAhIm-Xpg>Xlr`7Ghu1vxs9zcar`f)S!k#|H_F>mt#hnBD` zMG6-#qBjcAOAmvth8 z(mawZC1%`A7fDtwRY_F362%cIc)s|`>YcDKJYVe2m$*Pi`oSmBIVEw`$KjK(z}hl>uMahcwaZPo zQ=3pySxM4}R?;?xWM-?&Kk~&s$yZs)H=W7YOGz>VNvgz2U&tIO5#8FZXq`5hkzK(a zbBI)%U<4323WQ>w4#Y8d@(Ud|wV)Ni#`n;m8VB|xFfbu-A+QOvv4mNlCS4#S!N94T zj4A9|h_k0sytaA(n0MO8XX-ZPX^TT?7CUM9S!oxHX|W(k@-`4mXQiYZsxl6dzlLah zfYdUjt16_E+oUT+r4KxlQ58woS^>WFO*b0{T6CpbbpfpnERDMy497EUtAO@wmX0f8 zCYb5ptTTKtlbw|_U9*Ai*A||!@quhv!Lh*5af@(eF>lNa->MA%Yl%eVjAXXVlx&H# z*o<_{tjt)6Y|PBu*sT1jRKNYK0Je-0^0dgRtY}|gY_>)Gb$ktGZk;W#!PcTlS+sCJ z6S|*M6`NLRo0BS%+i8$X?3(*1BR9JxH);cw>)o{HGfGtpU^Cy zBPu^(D6bZ_pErt`FIkoUWjvo?BcBDWAfBi%wm_CV58|6s=36*qTbL7D2xiO6 ziY@q7RRA0)0PPgavlhXLd4O3FeN<5xa^YLH!cWSD%HxGa`-K^pMFmww_!C9=8$~o| zPRsT()al#?mdz&}{}XlzbUyW`-qcVQ6M` zBVo;MW}72ne`D$xC*eF{>Pqp5xjX+HIe0=L%~KT2GguK+%?DTJksnn?ta3;BReqbV zh=cht#2U)Fvy0Et0h-q76ILr(Mq(M})j3xxnLLm(Z)nK-tFo#ISc}v?x6|kOr7CK< zCh9s<6g!#PP>l=FH)6?Mj?m*SQB0M(YbX)c##QC3kPmg+QGkQ`ECHm-wF4?dSRw#E zMZnTCU;#RD4!e@PX|&K>t;{oGt7Ve}NX_|IDtM09t-GvdA+f4t6d#l#aq z>=s~Zi-;8jf3t;tL8!$7C{@!!q1CEDX)P7e8tdAm{Kum!pL>h8(I0EGALG?>+l)W9 zncTLS9*X%jM?^KZnc1^$4@v}kw1+=dXdG(ia09)P>MEU2>aOSjon3i+G!Rq2;t~Ti0`^B?aJ#>%>39@#9;^h z7=rRY9{GO~F#kXE$mtDOzs5=(Fv&`bV4+CK$?LK*$VrlTfMT$im*-M(wG2{{)zPZ9 zjbsUk*4Y5$ZkizhV1c4^;#aW*IfPYR+qof4*Y6}?knHkYl0=de30TZ{$qTnZHXCgs zzhUBd>Wvj<=^zG_^&;?%_nuT5R)o3)j1_TH$(;EDBv*D%ykCjpHo}s^VK$+Klp|Wo zTGjXpX1&3dj;WI)b?t-p>Uo&fqMm&0TXqNCxB~XsCrtD;B+}`xKnIPpUYSy{6HXC3 z26~Z%-`d}vvunmKv+E2uP)7PM)G$LQzjXsez$rPQDy^0!6jhVoUuS!g6SQ|6=#cgy z*guwyW1Wi!_I{X=s~1JRZe4CcROCF7I?-@A(YX1(1$j+pwa={Mhj&(zOqy9sI{GT7 zmOp~H;ix%SGce%$q(EVmn-t5fBSQ#=*jYjSB#{d|6D0@{Iohd6WPA@&y^xv?KeS+e%|X5*2Jd+ z->FSHGfo`Hk2b-I+`iH*3FTK?S@y=-hR7?JK)>wh0_a|8rjuV_T7JE#{0IT+)xqzT zBE2QiPi?z2+K1hj_3uIOJlfrH=7z@I$rww+-RbYwu79o+aZ@J$u~HOx)L;i!$Jg?Y zMXwidX=t$83z8phkJENl@8+Xrg3lKMNC)pPq=#TTO|uKZiTy!(PeI6n5vr2!|D{ucog?vYarqZWAJ zv&weS%<^tRN~zu%AF%-7HA(T3yHPgs(0rcWW_qqpE8mMtY5>k~K=)P|yYqNoFowI> z=z^6sfB?xv=vg+4+e?$uYZW4dmO$i_Z1?&qwV4DdWGR0ftPz7Lb6su~o|_NGczPmX z;;*Fhpv(fVH(T%!;6rD+UFro**wl%NWx1s)=*EX=mW0QSp57Jg|`Ig<}a$>3P=DZ;U?qVWi zXAKZsG{&gcnv+g0kBDA0Z5e?qH>{V(-wAWq6j9J7T+ zwl6vXS`M$-bVFFmXkii~?o94nGRHt2y0Z2J^+2VsUN+p%eWy+Nx4ub7#e>tEr$A(e zb_AYAtF^QHPtoQwWKKiZLe%YLM;s~K3l$n(Y`bt54LQ!csk~r((2e_1S0Ave`sHTz zvhQlT;kgFgi`Y53KDDienTK3?#HECOeF5HSK!7~zn}h*fkH$$lV|mPmgh8Fw#tGgv zsC`hzD~dim98&=0TZB%X6E(w_x<}?sF8XmCNo=tVKQ}6Sh2u!dLz8S9!-$Xmok8fe zXjsdVrJiN>D5lzW*nB%!F#X#j_rIPr#cRnLRomhB?;5>lnn=U-*xWtafK5Du4q(Df zVr4Z30-wJMn5{+V>U%Pj6z>Abkg(*y>4KE3XrBg9^Uw((Vpj}AOJFnsp#&HEy} zwn|0y1MOV6fGN=?P2;O6+!JHmSWe_DOsw?+tr1-f_d_x}-t>=HGeb!A2yoN-4B@$8?9wiltd- z|Eu;-j^4Moe|Y4`ct3)tC^;^6o{rH$1&ok;z99eMk-K_gZ>=wnTmq?Qz6FmlkP!d! z$nlQj-}kU%!Y&2NXurFr{_@C4JH~M?+D(wswL|Z{yY6_E zegLuvXFZ3BQJ*J!!NsEyDc%uBOQ!rvk)i03c01uuP$CR{yp4s4c=f#u$f8^xN6BbK zz4MN;iSe?#B6PS4bh?T%C5;y6i>B9)_VV%c@u4vplQmtAet8)!3XPT=i;h_Kh{__2 zfn7z$b;g)2`6|Uken$^Ujq!x|L;^`-vpd}rV+fO3V+yWfMM&cw^Tmk?#FPtrRN4?$ zSIX8d#}#$P-e*R%N5{5id9^QxWL$+j>x@rWj&F^QS7l95yNnYA#_v9l8)J2wP$Ha) zk!`t3;3kC+pb{VJBz_QfTV^F(wUJ$0PNcs~-R@Ef=i!D0zW z*ks9k)AlmbSenxK7Sbeaouq6DWH4pnbIUC+2vZ7#rxl{c=FYGZDI}7Pvq4f)cox-077I%DW43HcNI{Nh1)OZOqdnYp%p&c&yU7T8)YrnC&}Nf%6AybhgIgS#1@2O z79pzS5u4>PSQcH!6n)t*WEwB(+bwDjEhMcfR8xlDU+2=0=aH$TQ?8iPS5q4kgNPfQ zgIFAZ2E{)}oLNLIkoKt;_66{~keU{#nfmR&G|}}A(cce~@QqOSpaoE~0T_-E$!Dp_ z5$rKn>F>8xKT0EUs{p9UkZ5PiOm50dDay@7%PkGdU-^~WzxGy!;sCyWfN#XaBtnJo2iGvs$_T?shV}BOoXWWF$^_VIWs*u|U``y>4RuN! z@mD$E7NlxhP6NxeDm>K6eM91n7oacY)j1-R9%4~lVbL^D4H>E40}`nLEu|3snh&bm zt0Q(otLONtN`dMfh-!c$_)3N!1#fMKU+u6Thv+MbX9Lez)``%Vt=&W6XKAJs>$N=x zHQ%);DO?T9t7-6bYheiD0d%GqBdBZ_q(xA~6_t7oX=I#}XHi8cd*@^#f|z!^6c^oO z^0N)w12tD0#zup+o(zB!Qw#ursMU%Cu)`t_AkX2ahyrsUBck>Xp-xhelNkZz`(GwQ zaXJ9TTl9^G(#@y?BH`xtcNE-oAB9EIp8?RGe;pJ#f|aoexL}>~0y-KSr3a9g8yk@5 znU{;|G0Jf=2QyP1CK?g@Z_Zi2f&_>A`O5iP=rUp%Z&s@)CAy_ zXrUa?8EGP@Nkeg}h5*Jzc$ob}@|vxtGPO*Tbtil}7;}xdT2-e5&BzD zq{9!1-4$uMANd^tUcyImm?w7f5c)LPChcmyKh|o}S*y*lN*bZq=dy9q15~7t&Esn}=(`U$>MT@&&7QPn z^wv0Lrg;NIciZHvejM&6P53q(Yq#X+R`&f5!^n!&)MJO#mgt?weDw||c8cJ-p7r9}N>F(aZcn8IXExGRod8vl@#fJop zhlDsSIcjMX&V`I>9#Mcn&rADnJQY3(4v!2BD&!5T7(c>h7*@R-)}(r?E&dj#jZa5> zpx*7R)5fr20)g?`TgsESrs5;RP*!ub5u3abqh?LJwGpQ&0kOLgcpiXHPlNds=~SjN zjmL2OB~eyexk0NIDX%Rm?%KFJqRb!r-S`hcS+nFXQH<)vg0T$aMVEg>G2YA6c0aBA z*a4Rj7kyPr=emSmZ~j(mncni>*=imSprE9OZI#R(OL2$%q1L*|zGdQb206f^81IVM zKFeRH{gi%h`jphccJ<6OtX}s2wp#mlQ4Ig^m`B_`a2e4b1Np>$(fQ3ASoxhGk*NZ5 zl+JoE26Jax5IWYdY4~Gl)|3b=T^pz@(F3w{)FW4)jp!e?RE=-2TZbj>QC;PArp5cyNWnM zC{hyWp}uKpUd*fH#XwVtnGsr9m$~c#r^gZ-Ik_7^TLQ1DytpWC=|dsj5*Yf-2wIZM_>La(^s4#6tgEtFOq+@MP556Z>?Gjz*kU zmg*%_b4tN{EB`p}Gf1-$&mq~*>43!}h8gk8qwE<8^l7SD8B)iyxqplVsNK_bME@KK zSXeoJFy;_;|9xfTAEGQ=FgG9vbFQ>&g#>OcC;F?(ZI{2Ptrkqt_2Me zify&DfqZMw^`OAQDfpFkrJ{-I>v|p`eecxHOv?h zc(P$VYk9v+NmE&U7U6w{ib$vsYO4CS17Qy8K4y?`mAa#olHA6#vAzU?CXcY6oN^-4 zH*WpYgf>!~OAhemHwGB5V8Amvh1hAaMF#Z{q*jBjPE~N- zISddbg~JPoe+8jSlu~+vVO|;9U4}S)qsWahIO{;Rj%p4>vsqrV>wNHqYX z#z^rIS(Qs^+Crjo&Uh5XtF=#dXUayH3Sl93vh%wOzt!4*$cX>AG7>E)O(lN(>PgKX zGGh06ol9bk@h)6OOyZIfZLKl6r079Ver^{0#PHd5PFF9%hv?{EBLNPDeAIShTXnMm znnAgXJT?v*%9xK{SzMBAb)~SC+6Avg0aidnHBJ=~K5cHhd=R_{s!m~VLAH_$^UP>U z@&7i5W1m04AlIfvzlas;+puC6&sxMqt(&!eMmn{Q zMtIe)uhtmOqPBr$MAi%FD~{lq+Q9Q)?!OP*jFME_B*Bycku zv(xf;?>LNOOk`N2*PSh3Bk^|>qqY3;Uq%9f`;S2bb1DVyrQ1C(m=a{D(mzDiTB{!# ze!Th6In!3}J$?9V6T3gd^YNI=fg)qp;;N@$ZTLS&0zyq?rbifzlsyd>3a!5rWlNUF zh;7esZLy!o*m)?3DYHY*%JMmi9DufAK-Lf$u|brDyrO&7&6$=PPe$yeN`iM_aMntj-m9Io7@) zGIqXA_%bup`7_|#TZ{m{ZVB%JQrREeJGXfE8t(m+635THd)GIpcVlUDdk{J86`dIjITKDSeAe#H;3wF?Uk#paM{mZD$uMd0O??iqu{_x-R4%(hC?1Ocd&4z6j zir-3q3EF!7<0d8g<=qw!?7SiAp|qxO{|I(J@`dMtq_zLzCyUp4rsrjJ@HKh>`DpMx zOYo?*V^&3gr*{a3l1t=L;AcI5gyoQjixBO{o&;BZ$ml*T0Q8m+oS6#THtPTi3{R@g z0G-RwcPuX6dN|Dpcnj+oEp%AlGQ-e!L&ldvc`(AmGo8IF!i6#LYKkz0uL5J1%;V5s zqGox?VEBG#_N_(0OKQY7?8NtL#9mnURUb2d>LZ(==%F7y;phR=wZ;9 z6k2Nt{<0}X-#T`TlD)T$xrL3al}!lmvg4g1t}`8~)hv3MOU!M1OvY~bI9e#!$3B(J z5lacTQ`2@+%C`L?mWd=zP#}(GIr?3B2%L=UF}3}49N*X(#@8O*#%gc9>@Y?a-i4L` zVonf{OOUon=+pI~t+ZdgGFuatg`-eqWJ!aW!Bm$CyU_tH7!mK4>=(!EmoUOsFp|4i zl9v^d&xP$Tv+S=clRI^jXuOj6AxYXk@g2HgoiTfi6*DYyS)3K{x{^JBJmm+O^K6X8 zJQ=u62;AJB61$x8lr)ixHTAVu@}_Xo`bhHQ%xD+VR8HIEK-g6(`j`XZxEV2~EU7`f z^>Wg8GFSdbof?tnsj`7?W&!Tai%Bs`Rtb^j5OWKA}u1gG`5&WT%yMzw30{aYqYV z3*{~!Rgo;leF&;`CQ?i$!gyBuPFC`D<|DRj7v=1B1&Ek|2YBBr7BiA#JiF&r>N2Y% zw9C6hIVxT`yV4dMl$}avkW+pQF6c_Cj?Gr8%I=8GCDG3^@5+f?$|=6eaV5`X>C9-o zwip|SOmyY7jK^^9#k?W+>k`R)PadBroSa*gId`3{02|LWBG2-*&Dn@e@At|g(#_!V z$arm?09vs>5;6NeE^|EYH^5eqvIjm>&eO2T8@kGCvq??QP9eZ7GG{INq*UbDk>a*r zxK&x?vzMD-TjW+1-Yo*%^3B{4$v($`!qWg9x`lXNg&dH=#LWC}9hn3N3DgwXv^V*v z6UAT99e8fy_}Ftuu=0icN`!MtOckL;V?}#=MQ)cR&B|FAqV~+IrmWR6><2+A-KA>X zPz{yR0h?0n*isI|vcP=@&`l{03H-6Pj3OHPQK-SHJ4M-gV~m-@vkIrzrZ*e=Bcrc!om0Qs()7`ti_j|EBW4vu7ZNfoX_ zBB^8&sJt4fxDtxY!!mzDQOK+8BBSp=ZEKl%<5FB*MLQP0W|03`*(bZ&wM8WWaS!_G zI-@eDnw%wcp&~5gx+bj4!=2p?T`BOxQpT8HfpT0KWo9thSjebdO@DO~xZCYW&;P+X zXr(&!!;0sDh&$;X@o-5=bb zCS@tb^ErLHt}o_GUz3zmymJt6tBU#HI>TMH#P~%I zy*jKZw`}3rV=dRcTgIeuux3kSf@DG^dd-b$3EXyZU6+=my~%+puW9cVmD31`e zDvHAiis3{p-+iqBX`K*H-QP)U0iK#|^_q@Yp1?QyY9|O1>xh4?K>c~0P5PU}Hq>e` zZmn1JF-JJ4Saw8KJZZ3nk=Q(q4qA;)?u{;Kjc%=t9!rf5r47&b(pN8$)a3tb5<85^ zh_jS^q)vJf-mr15IrF|5a^GxA(2}RsQsCZFl-5$x+ET=0wCLC@nJbm%ZdIYxs)W*- zAgGj3tFI!Dydl~u57*Yf(?*Y8?{i7bU4SGSqm$g&HkPI{dapB4sv|1UK6|h8t+Z_= zA8Asmts%TM0cxkAte}IEz(8Gqk1wlEp#gObo z7PZAV(*^89P$n#Zg%IpQTa2R-00J_!4DdZ(0SbH4@sCXb2= z)WeFht@D{B@V0ayz7ttv6w#ZvyAcGG2LUA^{Tu6D9uVwhQlN=PzcvWy5%ZOT4Erpt zM|-{dOm|4Ks>`zs^GItD$D<8D*C@ybOTq(z(*whM6xATTXZU2`6oP$fi=LgXQ_1|P zFhXZPrpuRX44P)yL57{|f_@4D#U<#|z-w^1mWE!0} z`g-PwQEAaB3i=8vLmH0hJPYioByGDI)%i5q`Kb)4@MPTMq|eXt+s^2#g9xxJ2RYN# zaI*8{dAj9YxZX+GBvE@i$x(+q1f~06;^%ECUB(oi>);E`DaOYs`Qa(W{VCQvo$0Xs zXpW3&AI52zb>iLQ^wn(dGbAGcotZ=088VR>vG$prz!|U8Ui5((MHxe0tU|u`v+9w0 zDvzY{k#8h3W_2H1v{;JtbiQx>oatwqU;b|W_}%u|+~@am4mxvAo^vi4b8hW( z9xHRs<=+QmEA@L2MFr-0ZRbNW=EK_OBUa|49_M48EyTTF_@c9r=(&)bu@J6<v!xmxk&M?%4W3I)8A~ngOKmGl9pz*N zmrLdwi`_cQO}?7_8Oy`%%cCpH-yWB3RA?uT!1X#*4noV*<)FW}8z|r~;O60T{?q6T zHhbQ0N9upyZh-l;O8yjO{L_*8+ox4qIQ~zG5jdP^3sC`as_=Ilv=|d4P^w(}+o#0< z3{n5!btL(}Br#gyg#T@h+PvK0)0W)v+KeukGawTMB7H|0SGEL3cQ5G z*W=C@#-^L03|Ugw##4VLir7g9XPfW^e@IbcBSZA}3}M>{wo-3u4w~xji#` zuPP{TMVuLz>+(7&j96ks`<=AkN00%#dGiJwZ zfq1|SoQUMl?aO6llYcl;A+WQ>Z$jj5uc)x%ZvXFbkoEmh)6PHQpy#gTcRxDiLT{6; zar2x{voPjcgGb)HY9BOLqj|Xa=je8bzvgZHSqLs{jYJ*KaBMRRc_`} zStMltBt%|t2?7cv{77M=Gr!{?>neX6glx1w2oc)=28Vd`PvU=!&OpQU(ge5!$$mRh zc`2Vmc|!__QBHryLGX@)bkwk7zPvOQ?%Y-{1I$?eHeDa-c0!mLMu;%YY@#*;V1$T5 zh{))3;@b~=oK61~A;Oh8>8;PaNz$B*VM+84N9y0=APG?pAx!6+A5RFy{Vz)diGI0= zzL4ySQcOMjQY8HkV|_x^@6j2JFlP~cV9E>){TO?XTt_!)hQL_cj^OT8*-=i~s8tZ?L&J4+zR8ZqObmqSdP8Kpk&&7l8_H%##w|2wL=+3)oc&M!1 zlUj#S*?an76b2sqk!0ko2DftnYv(HYbCq<8Xf#2ln?S&GDdkIs;P z$!!i`;5IUq4NLtkF*4&(cqeO!QkPGaP%P{0rjYt%pU=~4Ch7_kWo%CuGS(N11e(Jn zMk<-&mkjc0gENJivn2{FKR)!~O=W+)EEO4JBx)onsPMZi5jf)i`^lhCoGL5X*V(g+Op-#%BO8GfP#Ayrq zpn%%Ni87m5Ureic)taV!W;JJw`C0UUCWMW0Td1J+CV68W^ZRlsS5oaY#m1J&M+?hf zEgi#$o^1V2Yx(*#%{DrYrZrJ9RzLWazE#JzwTxdL<2)#Ad7Q-sM%3o*ZlA17{oBrF zZ9F%^KJD*2ci7qX65rjgZK1ftBl20=7K7JN$DO-Otg3=3nK#(vU1|uo^@Gyf*B_#i zJGSe8skk5L13$QQ(>K^UA-Apcs9yJ>N1aBrY8j9waSa%Z9T6-@t$%$Y8f4}8rF1H# z&9FA#`y_fA#kOqF&qqACZpD~Tin+yfPuT3Paz>Q$uxRgb3+>aR+Yc&M_{e}tjcvo# zlLFsngztP?V?)js;`r|4`8O&esLyMzld>K31DAGQ>q_%ym_NA&9NIcEL@m1D4Q+jX zw7J_}INO>2l6a=;~|n6PV`WPWv_|2laS3l7JnWYUY_Hd`}WiGMFNy9 z{JBpqZpL~LwU>v5cD(A`i>IxuC2d1NwZ=SU?4W|^_GoKoVchQmVzr|8mVCmgBXQRD zkoT5m5gc4yQM&>B9vd8=W^zc!u&pY_WcI~57iJE;P9Q9cx{}mCux;ikk)ERrV>yiC)igq7&MKA`EyA;M7Y{jJS>eCi zPCnSc^aS)VwJ4m$CuxoBT-mqike^TlUqgO1G`vk8!pUVT#<^b_OH2QG9p!RVtvGVa} zQBdvoBMtkOx%Zh0!J9Mn%9meTxlg_)etsWs?P5Fn=2TAGBa!5nOW6mF{>Y39fkUzg z4`KYDZdpu=JbN|+Bh%+y*zYZtDt5J|anB`ZcoVsPxy^q{JIaxB_{ri8!xkIYcApv# z7WUkS44hhGo>%XML(dfsRF6i2P0o#&os->N?q)b!q5HS1YF|yDn>x|FR#?355zdQO z*S+m{YFW~USy@Mu*Y6ybUOg=GKm5F+@fOc>SL=4aZuA^a@R~yPzM=8?AmXE{?}M7` zEwW#VPV* z-u3QvRwNm;nVWNCRF~sy@`0kdVKw;+1bYY-dhqYc3Ko5$0r-oVdP;5kONqc{k)dMn;!xD-D${F0OZo+=zhr@@2`HOmU z8hIy$gi`~sMH|E0h`fqFglCzCSL}qXIa_oT+R1alz>P4xfiQ2>NWx?rf}+qj^x>NW z5rT8R-}sNHduyZL-f=oN_BMUfglf)O>k2;MI++CBV|(+4UEL4Zm0`>u?u0dHu2tP z|H0UoH$hB*xU?Ht&s||}2HH;yI?$pxcqp;lG*Q$EUOX3jynR{zQMQ~_QE~0>to4WzX;C)mt-t~JGR zC(FO4kL^W{mrik)9r`Tqs*cQ=kX$c6R`|s*MbvS@+4;psMGaT#glG_F(s@!$L}Lu5 zYVy8IGR2;Uw}i0q9*rrtK`)mFU0`y(V+_Gia@tNZC4q<1LLyCl{Gx#MkHP3MajP`q z6xM|3d#8Z+$sP(r-k;FChe8xQHPRHDjFm%@L>dxxxfSc^UAB=^X_DD^QUDMlhg7T?r&;FCMlI87-eP1rTQRU$#;(%9F;Sc4$lj5FO?C0*Yo z{dGdRSdhUlv!I_LK`yiU)3^FpW*N;cM&bn_k3%6gfmxG9nR58HC@ooluz>AB4N*?W z0kJ2T(R@oh`w30`AShcjFza|IyCMYwxly&*jSqJ<{ zh`_21A*n?nBt>Dr36Z;^7?R@9I}~gq>7)f3F(gz60n{&&C|~Z1AtWU^?@IDCOA6dd zU{Oy=OG(*X@rGfsuZEXSFRF7Ds(V#&Q)+2TOKIC;X~$h@CrMfNyRu%*vew~zokf#6 zH&k2Q(r6_*<#?^7%GTPhA0 zE1Z2Kj!7!d-c?>`R$jSP-V9gtJ4mzpA~1AI#1zCQ<1_aT_qx_~PP=Ccvxd0k*32&hqt0!u7TA(#$J)rGd` z@tvaktZ1;V3oNnNpa2&7VBL1sYJYkDmK59G0w~2JXkLoF!HRaeUjIodv%8sJ`=mDB z2WVl@WUdQ@1t+kCI?4igY75k60&2sO71l;kCZIVh+8Lzo8WvkXYM+hppX%0|Qvio` zTY^TA3rewJ$wiP>GrD`R^lM@BaI8XI;Jhu6ZUh;Y&`5)^ae5o(SzAj+TI@;dtDsWt z#uk7xfety0`hDGcKhl=*QhsguhJN?j=J2*E^gQAXPTnc0!m7r7d88PgmTFkcLeaEg zTXO~ib}%)cONmw8H%gKMPX#(~@vx&tzCvwl&*a*J1-TCR#YRxYM!K}|THlpne zD-B|6zb%=V$#Sw8*?^N}l|65&qsM^9p@3Bxjm`3apSGZPI*l*yU`U%lpn3#NaQa|W zn~3W}y8FZ67wwu@vb?MKq13ja^yQ(|Idxd{}p?3e|_>}Bn3#BCO!B3L6nWjGI7V%$B$>Wge^0 zo5!UuJt%q)C;ep3ho=J+!OUlaG=E#TJv{cGX&e2idURg1#poT_jNm-Jfw3nqZ@0dvS>M7IZppdT{Rky)_&t_|zVUB; z`S-A)Q0_D8x|pNv)((^bcbwM*(kno11k5 zgRkHC5Ax!p?rwG}p-&G##v|#&Y9%iN9NN9x4BW4zApgdmu=+x6dr1Z8JFi_|dZCo9 zA^*jm(2eTCmA>wzz(4U}_;cYF4ZRo4lqZSddm#k={+m771VC+)dx{TFKD@`v!ATkv zLB@&oXTDfJ76A0n!NP_&9#qF6pGBV2f?;)(AISCdf?vCAD9&(Ub(9-{*J6_q&qWLX z3$Qv$i$bts9XYCn^oHXo1_1?RYspRQ*l(VV=Apc9AeJ#9h)WH&EN0>~f>Z)n2mR}KAUZT>nFI&Z?dc@oXGu9w-w zyYpUnPtv?hiQ)m%l;0ms&W?lzn~Rs>J*QPUJ3Z-T{eHRZ5h z*5w=p6bat!$?~I%kUsLh5K>(n(#HOrRKpuAocm(RZ$g>k*Mq^kr;Ul=*CN0dZyBVx z_Z6dsaGrs<%~|8!L<{=sicoyb1;LtlhWp|vsl6rAgg6R@5dTss4@2HA5>?*^-!knB z8lYhDa3gj=1(F$)DAaV)ldl)z@;3it?BHBSiAs)Tg9F9|UjqW;)|qQ%uz zrG+R!Mq;?vkHS9-woPNqEu}Kr?13 zs3(pfWP&+JAWScEB`A@iA2UqDK0s9i`?{`=R@{mO@GXCHrg#tzm6!~Sy7F$0ViI}S z!A@D{pqrp3XZDzmMKeN_D+d?4i^O#S(i$mfq(5j^a9_EfJFGY_XD2gbC5ZjxdDzpH z2q$J=kFoNaai*llgNZAXXFAk6$&b44*z<7%!?HIQ*daa;1ck>g%jrpk{*>^sVqc&0 zGf1nM-4CV)Of+w15OpNF_1BB`U^fGm~w9JffnjnKyaQ z(L7deHKj^SWodoQvi9DrUo~~vM{FQ0>1f=YpJweY>QvTPJC|gfbb5qbTxntGm_wFl z*$(Mt=kp@HO>@$=6emFUGcHTGp0t)T=xS5Nb1B;}H8MZiq|>2=v@>nzq}3d(ppjFZ z-TaT~^_x!R!=z0X3n(*HyuVyW3Hj{d=R9_~<^Y+^_x5YX9?Qbfx-+j{JC^v9?y3>G zeMfwG!Y*gH#ZmU|>nmQTavL2gQ@Q)Oa%+jm#*81n4dTl`kB>8J9*;Hog9kQN&jXZr zcE6*v)uFz!z>mih+ zoq}`Q(Rc0ZkE5l$d)6~gV`I-QW5dxx?h&+B02m#R*8jjG$%7JW%x z?P}h#_`X_>#vo_w9uqraE^pFg$IdwrV9pMoLE+(p--LfQ1zZdTZ2j(5G1wvU` z2D6K6A6#dr_)DjDRi&U{JI7#0I1^i{MjeqR?_E{5vmJl>8=sGkJ{a)c6kN8)gn_c` zfrWIQ#bTasiIf7=1mzf%#0LG@!~@@$1{&}BY9?Eo?fMalOTQ-y;^*?|qzld)Ft5V5 zdoFJ7s&1k0Vqr+6{dGo;{Kkk5svImDsN1C+((4q`pBU2f#Sdd6WW`a?#n@@#yNtMK zD4j{@X;A36vMVP8h#o04?ro@5QLsu;2;EO#e+J=-LCffT+n6~A_A3Y0zK~ah!WBd^ z`*l7fH$io~#sR@9{79kf_4P|k;q-)={w+b@T^vKVMIw@sK!ljG(c1Ltryfs!Qf{k6g4W2KFX3^ zZ2=&rKp%q;tB9JQUQ-ker2z>e$37N`JtO>#*`+k39$DNZd%7F_hQ{`mUh5-B1#cm$ zmPm3J5`}6K#~u&|xdFOCDL{$|(Nq$;EDXM$bt>8}AnsyC7LCy3WC=V5tus^6z1>)_ z*_RI*VG_k(ghRb$Tt%6at)d_r6vT0QNiI2cj-SNhB`>}p3Cf2U#k__0c}3;p7-0NP zC6QGk8bwb&9MMUF!6`~q7C|t{AXH^wFiQM3uJSF&$4tjtLr=U&>e*Xy^TN=a?)PnoJq;U#v0^`W3}0`K&srZy5^ zcJsi_yol9;-IIL5&W@02K6m-_e%JKDlyn$!GP;od?KXXaIAe-CWAGN(xsc2sDCTm4 zI+udGf=0OPnvx7n;R?&;PVh(FPRZPF&O9{Z{^Xm+(#?ZKnTe5Yd$B-zGDLOe3cB7S zzip;EYvwX%g&+$;P_j{`lam~BQ4oe96ZvlecR<8N4nS(s*l8AHH;<-0dRhvrcJAAK zA1sPqK@V26LQwWcgB;j3yL=u>AuAeEHs%JLs7Gf`05KU+DiyK_=-LMhcESbU0nb<^ z=2`QY?{cB6Y*1TtX7ro~SB(44Y(zUClobv1F4vzevM)B5;f&iHf~oN?w-G5BAJ2jI!-5=c7$C;HQy++ADw(z$)yF%aKH~@T6T%=LAhZ+N z@g3+28dHF43xweWfw6g^=y@vdKzzew-sV(o2CT3nwB!j;qa_b{ zqgePI2w)EQsKf!ueH&C&0(nzvJe+;&Q;Go2d(Kz_*mJe&K|O`bZ4Sd~DdCba2fSL0 zta=9;(d00q&Oy=x(4m(On-`leX4SKm(B@<_^>V?BaQ-ANTL~$r4y&LkfjFFJ)0X7- z!dKAXKrEX@2}r<`36&#)mDDv^PH)PJcgk-!Dya~vF!ZY^{i@J&swldv&^D`{BUA&Z zs>y|_QT3~z`BkIjRFidABX3rdk12)!U{Bl|Ow$_7 zTN^CHU;4Yf{rh245S|y_3Qiej!iK_3*ks~WkS>1!lu|hHUv9R$UYe67+vN^KQwWEV zNz~*G`R+_T9?->X48<^h;+xr0ERcz395+*%KP!-ndd0j_8P18F7MYUA`+U{7Cs)gI3YYW)}>!t?Wm)U&+S6n-@jy@_h`I4fa$K+ zzUlnn&Takb-?4MI(sFZ5h?@Ipyrcd8dNH?$|An{L(B1L$MkD`ce(%_)NBVUiBm&cQ zUjSXvy59`*?z%t73r{xyOGbPnkVYe9BM9g7?nW?05Cc^(3HlA04PwGgtlM&5p1ye0 ziUXB{RA2H+_*8xHj}Vh1hPX&xa{R3r=&1NsENWzMp1>^2tz0~1qBH27ycQ;fE?Vq}9*FurMKhi{aet04{V%Q^{ z`sZ0Hl;1JcyBA0_^qC6 z5O{=O5(}k3ixij%3e~I9C><3sIWU&XQu%#J9E#6%$!rH#xjq{z6?a9}8gow-{(8>x zUK}gOzUcM5lIMNXe!TxkYTp+`@4QR=Un+Ul%gl`VS9Im?c{@xW5jkxS1xnX1(nB5c z$6Fn_q|8fxhCgc^L@2jgig4cGPN8eg*9y^CSD(s<#`0Yg(;xU&+T{oblbr7^q97u` zQQjgUqr*Yrn19O>d=S3<5tEobkpB72{j;L;f5v3U#qfIlj!B9pbH(Esb^g7gGhAOf z_d6!}v!bJpj4m?L@VlZTT4BUBM-$od)(k0fq_J|X(ZzFf<(~EAx2WBD#zd|F$3Fl} z!r`)+g%Ws*mno^v+0{9W_KNTeCrGH*J%%49YHzvxIwXySulzxgLu3&!%i4R?()erb znI14gJH|!G5}A`{88tTAJg5PcL-x0*eI54A-Qh}UIPo7u+lGz|R<5)7O#>N|n&d97 zu1|(CKbSt~yCqx`T4(adxNdpj|J(uLfm{jJd;vlU3o?_Uy=wv+FSeEqv>$|t@LIsA zaDv+D5Yk}e_dzlCC@;<@JTCDsbo`hb2Gzp)unIRVFhd*k1rtO~iA}N4dVdI7bj{0i zA))PU#)_91Ziy8js~HOQixv>VjzF_9cFymMsq8c~2X*3fgPLa}3G&5%ie9py#iqI! zzt~Q*{hx^1%ifFcyNR~#rm9v z6vzTSHxM4u9JXqN*?`Q)h)}I?2aXd}FN6!S(_+6NzF0YOnI#^v5hQqybRm+X(mpvW zj5EirJ;hBd z?t$jg&~pktPUozNv9^7e3xPJ4Eb~#s*idlgzOdm#2zI^K<0GiuOa2O}_#-G3RZ3uK z`4y9jw9; zO7&_2Qr^hZ%)?1>AMaPo}Cm(C~~~oVO}90FdjPYaeSU-SG{u{2Rm0k1%qeQ7h0nKN z2490wsr*b)S+H_91eQzbN^2l|C)qVZN+<$h-`Fll)fifA~htY!{g z@r)#03asbQ_=1f$>WF8O@#7n5SS+}BgtDQNstwXzOs1`_TFX>MkRllq8_!a_m4Yv6 zJ+#&e0xsw=4R4xC(+mY|H*CLNDHQo|4)&Em`$`75kG>E?N(4Ou$^f-ny4iVk$t1Yk4zVnWGgK z+$CVhTmYg8?0<`m7PIjht^kvJkerQ?I&pN1j^}nr9#+vQl>W)6bvvy7?JO?qvmwm@ zK~jJ|l5&go4y{7-Is-@0Hjd7Cic2|yNYs=<(Yp(KbI>&;QY0G>4SFP^P zY;4B07LHJ^+fQ|D?0=Ci-ilp+Enl&5271tZdD+wf{HPy@k-kA;_6DJr#U_NFxB@=7 zv5!8(PQfPq2lm{}z#A$%*5KtI)X++bez~aqFP|EJusFMZ&`0A*uGQMks^M<8bW{!yL$?Dz7LXM3}-iYo~IJ@~eezPf;iamh4zMIqV zvae=JKIGx#kt*xJE1On3V$o|=Tz>l8RQc=>sgWn}l-!}cQsh`7`(6glM!);H=m`fK zZxH?`yxyJVUoZ4{709BQ2R~%~`q1bR#J1)z_95|H>AEdgFw1fB8vjU#vpq%v>NHzP zeyQi|$*A$kc_Ay|%D?gPP%q1QB_4Dgdi~hvy5_tA&AdtBd>VdXZSzwK^CsHa3qE$u zbQBvIP)72AHgoYn)_(lq8RRP(@ z*F}Brp>KM!ppTr=^RQ0=upu&dBky{9;dpB$`P>bd?{E8Lq4?s{`{FiwgH3(+QN2(K zeQ|HRiFbXw0>GFz1hh>rscw7;tBug@jJ)jpOp^Ra)xp#d1H^0qvpPt#!CxH(KPCu) z0c8LB#tUZ>p5q3{6$0XI3Q!Kh7a$5S;tZf>0F%pu!cKsJkO1kXK<)WJEmN>S$Y(`U z4FZM0&$j;R$)Jx-L6-ADawtJmoI&h?2psYlStr1_m|(Z2;5f)LPZv-yZGZtl4@`-` zw~c7KY2Zr~_&Ed|1ib;`%Nx!Kdm$Ms{fCV`eLgf*T*sd)5YH}DPY3}D7pOlULTraX zH4z3=u~(Uf0sO-Jio%R;h<%9=T8hGSG6u6)lQea|qBH*3?Fn({NS|b30)+Yn#Gp#3h|HkR{mB#=89W5;t=6=95t}U@L`%sj=tqsh z_T0W1M3RFuxRQpz0{7t)Wz)7n!eQejnoX}G}QW(anZDWnzV2u=~MM&2tY-pE^Re9t!;s{ zyP3pz+qaK9y*59+aVWj}XL=iQ#v*OT_b=%)L-KQGdOQXh23;AR8yS@*KAg}z$@K;N z0SWq_DH(V9J~z}PI}7aFI9X@hSr-~vSFTw%DOq>TSq}?YPq$gWMeUMV;o`}A0m)WK z=}E4Tl{`r5Q0CDB#5*q2(--oUJA20Qa=*Ir+98O3e)-(6%=*6A=CC@kdHIH0#oU()H1x{vG?j#IMO)@F;b`=D z!-T8*?vEsdG(jbcgjKfHn*J}mQ0@t!CGHpp1X%9gN5g5rB?2&$`@DLDE<)h>4KCq< zn_zymM0fRc6Yld;H(}aHSWE(&n|yg+!%A8^G>=Db;f4<#A$ZH=#-~EeZ9$M!tuH=V zo5~rCG>IyXgC>2N-FyPv(gmfO*AfYqFH~b7!XwnoSM!}_kHXRYlFx|rtAC%J-OcDB zC|YlW6QDbSE7*b!jluqKhC1$0OX=G{TZ$`$L6%ab$4EerF90&Ls3GoZnEIe3z(ent zhU<*s`q`b}of~#ksR#QkJZnBU${jmq2^`JJ6MEn*me5>N+w3;ioXV4)bKjg7-kd8{ zU1ZTxG6K#l<$_o^=VP=M9<&r6&?l1Wl}fd$Z!}0wVj;%1x>>ZAx#M@=0~aA!e0t4^ zOIR>VJLzdlC~oTuyS8!CHr~@Fr~po2&RF1#yLX{0bPt5%G?#W^ zc6A|wyCCdcjAi(+DG$tLoFOSN7qj@g<+n@`xQPc}Uq1kMY};|!daadvh3tE2;jz~R zz>*O-?9y_IWbDe8IBJKW?X*6&L+}{JSM152k(oZC^FF%RKJLkOf;Krz4>s!voI&|+ zOojeuU2RMEz^&5$`jl?5HvHWKkeek=AGGQ#4t!s>^1#LC0V;6MDHGU(4BWBQ2@6_* z8-v`3T|5!^SP#H;`62X;ZrccOB55bW=+}tlp>Ie-=Tt+~vAy$5gXj-mNhf=u6rFm~ zSV(PvsA>2hNT~XO>F}N zUBk4@WAU{>#C*^KS>MuO-->p>-ZE%|8NAijIW7&FuA(?i2OTVn)wD6!@eX--U>Br= zFg%7ZV!PUTL2hKIKCGad2>=?n=TTefXu1Da83;KNbe=vj*gAo#@YTi^s~Zj#%8J=3 zHQXW%!hfGhXf>c`*9KSty@9@;M)t-04nh44LeFOf4S5W*SP4VRnBhHfRwMd=zWwQ= z6MGX_hn?7dY1qM-Gu01Mv1HTT?t`rlz;8StQcuuwCl>6k4Int%D?PZ3Nw8Qt3p$+z zOtwGM0a?BW8J9~7A2Eqq;e05c0>*Y?YBv+Y4=A$CA%ka_gT5b^eNSo|1yQz9VuE*L z`ut@;jx1fO???0~XA^jan_FkdGo~_EK=v|ofjYAxk(0XF3uzgx)=LZclf%yx7AhU) z+T-R}x)wiTP2qv(a446eJr>}`2v!fjBX=#4Y%FDYEV8Z;a9BbD>W~|wU%f}{O#jX;(U+vOaogVA` zJPKaY`JV6j9R;())Nbb96X)z`>SAowZewgL5+^89u#SbPfd$w7JuX%hKaiXWOpcqb zgNyIQPl(M#jE#$t*-tUK5#O}xGDb-K1WM#t6tY}|K6cr#4bQCivg&PqKi??F*?g7V z{$gx{wE{dvI&7QWK4Xi^#mDpR8zZmR68f`6!s<=b*sYq{t?AuAwxuVG}ib%5Z`M}wWb zscpt=EZ&A4e5l^GH}*CW<+j-vfllP^cizn~Ymm?CTfgHM0pGeQthVE^zzN@`BD}y+ zne#>)Ke6rlviU}xA~)+j3EmunKSyonT7&bW77gtB`^Io;U+zD#Y#s3s7#-uZKLL}h zwJNYTn!Q%nmwyuYP7IKwq6!^XML!kdg@n|>-+&)Av&cz|zrF!G4sjj=?zv5DO^ zzp$FdCcCAbJt1s|jr?+EC~9MiuiK|=$wnH?vbt98wf^E5U*~Zd&-WO6V-a+^*N=US zM|nKAf-gumh?{+a3mv%`3KUB}|inH$jBE9*^ z`YbZ)ObL3rMWA;QsRR0gJ*d%kd{J|hqIi_}Fv;}kJIDCuz`@R#S358NE57kOeX8jY z%RNkfFt*&PiXHZO+Am=_oJB0Ko$j90_{U4^O||ja@w7Ft@5O`lDgfBkABC#-oJq#} zg+SI!^Lw!Q*Q)>-kk;1|kjmwy4nA=Cns#HXJn{-X8ZA0@b$W6D0 zsP6{e`;tQzw*+gRKV3C>{4~nulymYnEn^7N4&@`FT-o>?Y2~4=;EhHHsKn+L7yqs* znqG8Gp~>c#aqKmc?A?RM?Q4GeM_J`|xvQLqrXoJv&dP1K7dS&-e`r)Me5k(PFK!^F zoLz>lJ+68muRed;fZhtnu5k9D=?cM1Y?S|ajvM=I0Tmt!rGWDIJ}%hcI=(W4AZW*a zZgRo^LH`uBKllH;sGa%77$_P+{MPy}Q9Ei7NMma2_n%D&Z*dp(vt%}nZ+Sy(bQ!7K zj>lWKu^(i?ltPBP8Z!*N2_HCHwpkY+!W$dB8p26ctD_1aT z-`APGIfVoR`-y4O&bwa36J{ODm#_`q&eGwv?qIUdvp+L#j$bK%pSttFnHQ;2) zI$Me34m`GIs~r-}1qB~x*(79sXy0DWWbRr3MgkZ-l_+?0L>=ihUXVBx$eYYM&JC-3ewkY} zM|M)z4}W_lt{1K1q+y&^KCNMzo8YwA5|@9aX<0klKxxrdxhv|tN z43;(1s;xA&b58eGsdxmTP>*)jaI_NuArQ z`7VB5Jb(MVQG37Z)tra@Hy$-^`*GoMVuwj7)D(wlB~Bv8SuN*B$9aP!Pp3tTYZ8RF zJKsH>SKY54o!9*dy<9fKIiFm%;?=xdx6_=TTz7MmyxjIn8=vkrm9(FpS7|gp-iyql z0seW9fef1(b4O_S7u*x@&wGrs5b^)goCU)@U-M?y1+DOo}jwo=|8%qk_i-FVh|0`2tFx*o{EJW_>lO^u?(ZQPByRX+s*n)8I-bh!3Yd*+y$@7@d8+wor zx){TT8YT&n-bX^&B1%VW4&@Dgj0b;+P`OH;tQa93>!zS-s5%IVSSYVwp4ZBhTJhN( zqCyBFg8YOEBKSKFAK=uEN{FzbNmH09h-N&3Q%plE1S$Z?6EH?|kdoJC$lD5wu$qky zRF}YxskFMdWp0MBO^!`!xefrUV>1J zqML`mVLnwagBpP}R>HTTBE`wiK2& ztXub5HEl`qSU2xpnIF~cHCRylV>lcfw)4zfFx*{;@v(c)cF{>gm7f zmcpjS{=_~1n{Mg56JJmS1nD1BV{KZKuLrxZ-KwYn@A^>xeQKAa@M+Fk5>SNSYfJwA~KWhd>i;W=R zNP<3s3~>FgzeOS$w_&1xZ|e81vI{r6h(r~lA|Bvni?W~e0ni9@b8$f6u@-*tJJBKd zT7siowxlLd{06;4D5>~f7*c)kZwNN+L3yYX0MTGd&mVt;kE>o zU1OO&Uo+clKy4VDWn{h6Q zBDbO~wnTm}Y%VF!ztWRMg7ugY{}oaL^==W9Zm7Px@2+EIaAcXGr?9#ovr|p-SebFN zy+%lS4@DfcxDlsR>^MOH$>Kz>CQlm9v^n)>K&UWl-SA=}RNkNtMFUC5ymc{d*SGHZ z9&3mBZU4EzE}d^uc}e3hXA}uqL_gb-_Oc`)M7hK1g!W0auydF)znpGQI7-z^V@E5?TeP-So9_}&Z7o?(oF*b1Yu$R%rTU>%@kaGHA>!Pf8Ar8sYQ?(G9oSv^BSGW z+uo~FIlB%l{_IJj)WTznCN7dl97-7~HuAUSTBlPc_y3Z+JWmgG`sR{(o+^B}ftSk8 zy|Y`31}!rbdcH8>`f){Ep{)yMuBUJhsf&XO@Uc%}0KaL$(_H@y4$!@Oo)x9CMv}Oj z1vgl*wO@BZBaEmVFO|Q3fFsD>!X|>-tCDx6ooJcyGAPwB2cdeJ-v_IOL~tFFjSi~C zlsSy=)GTd&36RX**>}(G%R*bQ7YcbFYDy@}N?3l??!toeIq7bBSGKX`_taSH{Yrr! zxNS$~m)p|ssjdgoX6TQ5J4%t=$oV%Vg|gLlfW~ zK#l&yX}0+z;0*n9H81Ey+yjOF6}ilv)mD1WbJYf78{MV=52A*bGOZkI{cn|CX|pJ# zNDAX-&y&lxaeeE-y);u+id*A%?9Y{Ye$ZMqsf6-|XRCeUQ|~;Y9S^jQ6Wn=mzB{c{ zCVO%zyy2XHIrAuoc;nlRM&KJVMsX78$EG%6wvTVBYbU7FA7c_DBZBA;o$yH)29bym zd4lv@)`Q3z@V^9|2O3j~7MczA1j5|(x*|+YyPM%I^?=w2WM7v_9ri`Qk0{=&-T(QZ zgt0V;!DTzuGWDkb3|`oqPW$}b%tQ$dkiVVb5tS&A$pOE(0|`)dSja?%ciI(*e&mjU zfVfjZIUqnwVLnWQtC$&6@x?eBvVUPFY=%(sLNu9RW-mXN#DP(?ToQU=UQ|b7gdkFQ zJP?zQwlx=1iRLIdVN{t?SWuPzlIDf*yd$1P`9uQ^v)F6QaD_mKUTN`(UaZ`BLe)_?`>)d)az-vBr=39H~qs z%(_*f_IaptZ1bHebLs!Z*;__M*|7iG%nY5^(A}ld-Q696v~&qbDka_B0s_(?N;e2H z7<6|^cMC|*JKp#6e`@dj?svV5k1S@*7v?(W{GGq!kOmn~{*+wJ=+Wv?hdmkZB7U2r zofqXIko*;W$QIfQcM?nf!aNg4(hr1VQTsrac(_$Ob|JYHgiqw$#{^iMbsOVs9h2HA z@ius1Vs;H>BlJuM$$scIVjsrMi+q4dWFNaEcVxXcqm#u)P0%465#q~5Uhc|K8w@F~ z>g?r0xOmdMr0T|^hPV$QZcKd4;YD5h73un z05#1OQ}$#P*^H22ft)^{3F_t*KZ%4hS%CZIT$bug3tZL>3x|VfK`cs0yhq7F`6tpe z*9gza`Iw@Pk#016B~@8r#K39q1`#-GkZ$X8$zTx1hf;Mx={agZh%W~TKI zD;n@AZQST1Q4V$;uR!g`5@((<0gmRyd^Kni8!$X-1Fw9Ht_X3ny{YbLAm?(_N+RqR zOphFlWEZ}P#_P!K#q99`B8<_C*H6pF12y*Z`W3%ynhHKIK+>TtW(~$As4pNnT&0`R`P#?8 zSH$8x%;nW(n^M|Psvc+GWbHeVTDp^<(3Pec>N{rHb}kh*R)C*oG*&k4S*9%mF3A*` z4!RuEme+~v?<{6ropuGKU&isAeNXeCK1ZtNl3jGD?})Rac~sMhISdpYXfa zW*KUB9{A$-PE_wb!1*4atP$;qK8F+XFT-~1!{NH;UwtTEhREZnTz%G{pFQYb=7KSIwf!roimu_eM_ zHbUVlLKha{E~MgV5b5R}Ihr1+V-e|pr5Hd;6hs*umVTVwhkA9pleO#}1RR0x0&ukpmdE97k+<3YC z38vIcavD=@E>I*J?X<|S5D&GeZ**(G9i@u2o6n!N2LlIrV0rA ziW%aId`Oh`!Iw2mly|^a#7I6^jHU0m+MSQ!mPMHg#_g+SBEUusmyG{*c>uW)9WlkvTPFG z?BcAr(mo|b(}TMqyDU1hN?Eq1FS}0o9kUN}cs^&qTwGf#G#tq37S8!941Dv+`Ibeb zO&C`h{Vsf$iFE~x3`2)Y<&p;H^g86sX5|{x=E#k`>t%JEfcK$K`#=}6a^S9cJ#)F` zLRnRh^47DUdyjJFh2O*a@|tIJh8<+A*RWc+@-__f_FxVYpuDfF%wuy5tfMhUS=s1q z`LsbuR5BnGkb;6mey0#KtJVv+Q~{L=h?cx?d)2XNI=}cj9|Ht_Tv0f23d;3!IyW+pN5wu0XUbX5A>^Mdtuo5U^Rl!$PC-Z#kDphoH)fbPe{mZLQ`m0~>RPSTf zxU<%*s?<2!*EB}g*q7JT^wn7H)Lc{m_)lwz{c1_y)snZ@l8Dxl(<#zY)X|I9F&h7? z37*yMH#t}@wWWBh8hY+tY5XtKJ8qqtfNX?0smpBd{3EPuAOPLC>>R2qZZIjtg{`Is^rTEo@IaG*Uhi!|6WD zE4Q07bV@6Cs#tZ3MRzI}bqe)$3aoeXVtmmbs6T(fb9S5Z+nDFNJ>~WV_q|^VJeFhb zV?88?bH(^w$Uv*r01Mh;m)ltvu*i%r2793cBX(jYJ*Xx5R*Xg;LcxYlhh0s&0Ye3M z%W-!zeohQ6>?Xrz=D>c3RpE&0?!f0y$W56o_=)qj{WE7ADH#AJ%lt|S`V#9`-|>SE zpS(rxGgSTV>xxh?Nnhe(uOW9|b`bE^grEQcmxv=u%>jy>2_q@&a6n0g@%tSfEFTCA8YEjCpphPs zg!T?B0wdT%#B@V+&_Mw=+_z3c@T70^YWR!T1Yfc7SMKok)o^A^hUo{1%RUY74i27} zR2o1B@C!y5Yll&jzYY11oPH((u|qF%pk)U`VDV8y5UwD1fBUCVtb6D-8$L{HbVqHd zi*4+O*qD49QSHGfFSMWX5a>A=!_*#o>I5(y0%)et4YgrBw{cc@Jn*;!`oaWY&&B`9 zHbEdg!S4(Rni72Qvi339q&Gj*tk=?BXW!0izFU;+(x z?nAbiraYaNzMlzt16@&@!m*fnqYjOUpRmoH##o&wHyO-)1JF~=k#f(G*Uk`v=HLf& zl{v&2Z-8R;KK!w{QuaAJx4tUpQIyl+YUg2#)sYu$3xhfH*wRDIx%m0H(6)}=9bg zhPbeDS&(~~dTbe2ed#EE!EtPvHu&d-`1H%#nY+(F*+D<)3VxC#<0jy&$SnO##a>2t z`RQyi-L$lV%P~T6x`@9Ffb)K02dpwBuWod#5}y8~Jet9z#;2E96IdcHSe(3$U$fjJ zju>3yNWiBahOR8)Kqc0%28q}Fmr136;+w61yT|Kt9>K6!72$x&kbC}_=>n)sar{>sWOy7p#Y?yj7~9!v5vR`6bPzZn<$`4E8Re~(Wf$~GmqG2NUyI5d(zV3S_alR%vcAR0~p@*ki(qx(;(Hw@^G z`Jf2?;jTXDfJb}r0DAz};`p~25qdSSE&;`q6y%~%qy}MFQxbzE2}bS*qnyRa6FKS< zcKBc;8zv`TdCy)BE(=0WPrsgx!FA3gLB9?eevNi_(j^keKj7_foq9(8YApPv1>0fy zxgwX?AK!6KS9`7jKYyb!D>$~i>eNUIXI)G^S4*Bi4miJBkl=g%lf}GS0(u!>U&aXr zHxesYi+B0epb_nM(0?jnyrDNSxuiNpI}V_BeJM#t`wa(lRYG^A>koBO1bzpj)s7uz z%z+TQ;0oJHTg|cX=g_(KUs%CtF(W`+U`TsE(<{&4;?iHTM_y&pWRcX^S^9Qp{30=d z-Yn03rzk)TIKhhxylL#~dTIo1eaEPI@O2seGvV1-a4o+gPghGamy^dgLXagYi26N^ zOLroEe+^CK?b=g%SkI?h3OML)g!|>Qm(?uXs^trxyFCHqLb^!q`w0ZsjHVSU__hiC z(I*I92#m_KyJsLTb z35&mc8=fzgZk!dPHP|z|g9FN6p@ycSkC!N!7dGW8~1<-e z8%I}ZbcRJo8jL3mCo5m6s5m%9#bFx;EcC(&aiT?VQy8tA(p{CK{rpUo z>nov$8qWaYc!6)?sfW72g6A)Fq4hKm4Uzo@-2rlxy4rz5%+wSWDF>|o*oLDcs(e5G zD+0&eDEN;ODk-nyVsGugN~p@I*qV;T1$DHlA&-pvbN)pc~uYzpq=EhB9!P zs}$74{Po(8&qp47Z{n6|r~@@6tnY3Pf9?MqLD;t7NT{s;UP8qd_)|hfMBrhB`u3~g zq_zl^7UioQI8GrD~)F?)O@XCc67@t|!<9J2Pt@K&GDIvDuZRj97)Lqu3 zB*Z`bda_aglCs&v*y+S9!asZ@w^UwQ_Co8jhLMljiUI)=}6BElXY&7A2J zqD-Oe=YT|HC-uNOn*AzISvwEImDwilP2QueoY#BTiCN*~zPkDvO}ngNJd7$p8aHYN zfQ(@fqn(Fa=~JGRM;${}n3zFsQk*J-WDG6TD1M}%Kn}mIkj^0n>ME1{;I}KSy1eiDA?M=B zs3-i2`B4{gNfS#4IZfEE^fye#Qwu^#r?Yaf&!uLgCK2DPJ!uD-V-MOzNlrt+9$}%3 z7@z;y2t?+@-ImwXi%VMQbvu_bCgkwk*M*TUvHP&_e$C@ohrGDc^CfX@+Z^g?dZI0(~{OA&)5$^ogd&M zA=@y{`{~Zq>-(!sH^YFjMfO_Ev6-5)4c@I{hDM=o_HeaErHasEpWT|T-|Y)?+NNHV zS6k_AHITd(cR1);PrC#&@!dQ+oF!}|LKg$kMEGH*?p?^Mpz-S1+yA%*k68*g* z`X*Qk3!r(p&U~v`3fU|=ei)Jb!3n=T{~3GVa5nNB{s5aMW0d~6{vL^ns0&0sD*L%r z353_%MHcN4j1y-b#P~&T5i(#}hm{z`ib5}a>tw8H9D{~F%S&Wdg@-GgxLa9jGK@2J5@l6JXElIw zzpONhu)s@}61pRmqKIsIVpw27fShfcY9Nj5CP|@5hc_v!q>quz7pzmCUmTI|v@P*F zn=#n3LGk9kKc$3#DUe+82`bqArR0Ljv{+&PJ(qhV4~FqCrSLwWzdQk~Oe;6(%}8zJY{k}qoN%Ck5gI9cQ&6@t~-q!2XeslEc|n< zLoU?Fxw9cGL6gxEKH!18hXNMNxkqq@HxYR6OTgt6OV=6!oxSf@2+E5HD`>*p6APR| z?ENTJM&B!^i4#_q_i;RCix)^L+UqMf1uLjM?#VH z{)cVr-y?AS+OvF!8w~NCDf&wsYpM*5D-QCB6z=mYgn%38lR28|Y&ZEo1>9Yo<`7Y@ zSd)(g6UgB~{N%HtoZiy$hTfO;e+#(h$FnaR%GE0!pA8s|=zK+^Wd)fqtLFB@-8t*K zGQ(M44J<6*TsC*%RJlf~F0PUPZs}EbdHF^QG}js4^u6}*(p3~oEpx4HcBtyVZChp} z1d>un=(Znq;=(Lx!=tMIO~4)Sd*J88&F?Saq#Cwv+NQtAue#7R5bkV*b3>w8_E!o= z5SVR69GGpx8m@^D9JDwU5UR5bb}>-u`^cbeAOzuXjr=PDzxs-oFp9%;_nrWaY&@8?yw{|-ZXTbn&P^b>fzKtPK=WL)~>hm5l>2)nhoTM@Hgo^Xd3qm3!-NT z+t$%hx?+wEne$w^>9AYo4CNMgXgIvj;zL2|t%vQDQr3`nRo%N-J{bP*^5M)@k32Nz z4RJq|VOSP#AX0o5AHmI-xPa#~pz;|3f#PV=uxH8?mL<>ciE*ONno`9M2f*`Y$R+?< z#JO&-)9wpPS=OmAZP@EKznN-be)%a&W_Oy^SF*8mu+l^Q;P^;HrQSumhK2XQQq;J; zM_;Ad@YjobHc{@OfTpIR+%$*k()sIF0n55dXFHDt&pzAa7ChM_cY@nR#nUFouBALL ziqfT{AN-#u(vAq{+PM&E_{ZO-gg#ag%Lmtj7GRDOZGs$D)GyfGGexd#T%T970=zZp zU9zG-maTcT3jLs|bjx%7%tHG+r?Z-ZN`z#*I}LUDnog9EJ;sX3#NieRcV z9CJ^1WOZtw_#(nb+|BYheQv_+YtszklXH^ z0HI+!%=4namEHHnH({-vaM!voX@?#^KQ9YnTv-p0Ir__; zxEfp*FHr6(lJKTEa3wup!a7if`BkApkneP0>SZ83evl$}5H_nVF6#^QavRJY8zOIO z)D{IYds~WfTPjiqlHM1<6&=1kA)!IY+(q#B)DY^r5c#nX4sTPgC`F!GIzA;r0q;;@ zQv7{ElEZCs@mT`Noe*gyCDC3w#a;qsgAmo7C+aP9nk@v{v%$Kg>U8Cf4BnwWc8*IEFf<5u=!MF%lE}}JJ7*HS>ux*KoV~uzo>19FUNXZ&q^2+hTF4`t4nz1~ZLLrLGBI>1f zMWgKP(gb6%^<$S_#geAP z3e?7)&BPL)###l(d@P9LhsAAg#}R?zE$QMbq~rN4;uj<12@B%gh~t6<;=Wy3{)lp^ zDR-&MOz2aLwwpZ3Wj}8CF0g@4%;-bL?|?6u4X|8o? zYP+Vg(M)WHz8r?>Keo~+(9-eg(;1*?`LEO4Q`5gSr<*tgpS8Ts^A2=G^oMLRM!^|S z`V4yMjC94fDB&4Mq;K7?L(Cl9Ut!RBli~aJdHe6GTem#@;?b_Q$z^>8 zRLJaGNIzfjxUGnW+`x4=cu@_cqkEsG9N9->DyOn57y2s~jL7EN z$a6D3YD|V1*Xqiqt53J-ZioBVF3;~0Skd+o2o>V2XDyrv>X$jzI0(d4;V0t7Fg0_tCNr`c>AmGcZ z`KbO~D1a11UJDt!;ISj3NQ>YyyJ9uhw9by!k5?f7uBmSZa5H1^;-6IV(c-0QA^+iL zxndTJ(pwA=QBES++~VP+63w09anb^p*T~JMc-h<-@lt^CSRH+PQ+Nmt6NC-uLTS!N z4x?x#cWv1_Y49Iy1KNo62gpyz+C}dkXb1Rc^AsID1r%%E&m5KO;$I6 zf@u7%cA+Fu5Qx=uM&zS5VxcrAwJ$v?B|L3JhduTr62%DmGV6zRnuL__5m}5DF9IN4 z{fJsNO0qiIxwnt-#^WL(+*Y!NM-q`7(qA)V)Q93p1+j1|B4=}Ki1aA zu6`>|9S}%>zXLASprq4LmXF+zgoLnvIZ8pMDDWbrI{l@-@3~@WA9SYswZ`vv@&zF% zT`2uPw|@}MEK-A3GE#F;7lL@L71W_Ms%+Pdmf8hb1#q>@J7jXi#2a`C=+v5H0L4D? zephNRTpKyH3;%EcO&p6t_C*&4?RP`Br*`gGn*%6;+~23LU}wIK@>)H1<1IeeL|Y(` z3KEKoNu$_zf@d3DMB2TjJPnn`4Ui`_<9ju1Uq6d!AxrYGoK*~rw+-Y-54zV4z*qXK zKNGOX50KLhaHI_4H@6pJ)n)8MK2!7a|M?%cu1otR7@+bA$L%udgR?mU`1F2me7ihbA& z6(3Vj9wTONBFP<70gaJY4pA1lt53 zoSlGMoL@doMqb@^midZJr$khZFR`#JCPHRL5IMO@U@BXWwD#~> zrfZaLcykiF0gYpc`LV{D@>3?FlUyz{x*OB=%wMZQ%OfJ4>wSmovTxR%wx`R^aNI3gC>q3V zXD4?@wsIRDgbRjkixyyT6Yh}QZ<|23QR#NkKpkksyM*z(W){0}q&;-HJ&4>MaXdDR zmbe0UC#hX(7`2Z^eOeF*o#2VA*6i7`F1161ff zKkQ)X{6IwgP_W=oDD1HM2w0RLSUmzhE$=l}9nifwVz}R0&jZrTb}3hPZL7xjs+KD9 z_6f8%uQarlhe!Qgw#Ur?P&IU#17JAPxOW-N3xFOkLkEX-WcDU6?E+>(HBH9-Cz z$_n2h#aj_g1Xwi>XHLu%txhFJBn36c+&a(268E%@0bR*G=J(^8iHP0&FC6akHfm}0 z>M@+-3rO`boa?xEaOPd&dC}-|4xe1{G7p zODz4;dk;8v9xy_$dXFym^Dgv{eh)pIwjy69;2~arD=***bqt3cc6H?On<{?KQIjA) zQKB&LEGLklEKuUZ`!N%|6M*}7Dh)K}=gsP`8xrt8CXr&_eH$>=lD+7DRu2lyvwz8;t|d1-5tE+ZbSWE1a?16aF6eDkAm2R$Ue+- z0y~o5mY=Wpz=1=&-s8mel`qg=G-7+46Q?xz*Z;N+|Mx1NKYB2|j1tK|t9*!UxPz84 z%p9tVt2UFYshr7eGh9qIhER&|BdUCJ6Wf~kIL0piWPi5d+%J6xvnCPSaP9Y{6nkzZ zJ%u`rE>mp{R=QPxw&4Rcdp#c^Bf0M~8xM?I7(3jNDgN4q%ePG}%pAj7DfSsk{AhTH!L)45M1SI*n{{n>b)U6?EUBh*V{xUTSk{!k$n-R^;ZDu0VVL zoaNZn_uJm)l*jM`r&xc~AycWi>cv6V>HYLrUp9pH6ZdrejvS>NByAN;0=s;;8gX}o4b0X_7N-pGrj7k0 zjfkgC9;4kz;z}AyPi)Gy2G1sW3l!N8RdqsHFFc=y;^tz=MrwVm(A7BxH_~bwrxPhh zA6a`%H=8lB>YEIyrRoQFNl&$!u}pZ;n#DG{OXmmJunOL#IUg7Vr=(xbQ~z9u9}9Q+ z-o*Rj%cg62D)Jh!vK>*3tjT@1%?#`olmEih*Wpw1I3;&_OeGeL$olfmB8ovrL4K~% zL_*5J>lF+X+bb1j#WU_c9~yySi`XJ7;zzd;m@n7kE($Mc;$#IlK0^xj3}|U|1zM|- z@x6sNV*0wRY;!ztv8?llr~az)`PvuyRP=8art04?xHhT%W-We;sPe%R@=9;^N@kR~ zEh};3Zdj|9DE3}_I(^*g^y8C)pL5f~u)$WL?A3M{!D4B2g5*I4RxyEA5j3^WWvhU%iP%2W zb~u*PT?{SNv1S5;)$i-0{PHl({BygXq26OCYM^^xir010Ik93Y7I?4Cyu1xwRNT3X zQd?tKMd})zew%cC!T98Qg2t6??Mp791>F(_I=s9TwKJ{SlUvyw4)L>*&dZ^9aBTT5 zn@1plxBP+w?P<*9-fukqVpj*+FmxPN8QF6ilyII63%oa1UoAmElTwER?cBZo1qo#?k=MRSiX$8mScpxSCRhUri&fXPD#1R2v)cq_>(=%l0;TQjly zD684(q`up*W)Z(u){4+cE5e?GDEp`&3FD`KButT_?Nc$x*i5f2wqbYPWzk#KdE35W z!};b~)f`n9ab58gJK9Ihnt4JE{M44W8H{45@as{1nl1klx03n`TitgH=gJ&7V33ph zM8>j-lF;`L>h5{Ex#I=-!ZRQGR)ci&ej#tuPh1Z>p(6jQNQsD#kOKdg6bb+Tk|On` zVv-dS_R4Nby?tW!?41GBGS=oVq5298>q(}u?4H}K*5w`1rCb+7I6zOcK70yL4 z>C{@kJ(3Ul$|rPVJ`fU1Rl%mt?X~l#NU6UgVs$vz;<3Lx)Q$i`cKu0_KDV5%_ohC1 zKK!}$*Y;?U>ifS^q`y$6_dAD2D_>*bG@6~CuMwRJM2ghe;g^85_T{~%q*z$v#jpMC zcp5t7a@vdLpFzmBL@-Gt0VeZcYyo?N5LDsD{7^)SG=)tm?X49^E;C6NMunM%&~>^+ znS)tV%9DdM!ppToIE+}W%yq3riHQCgNTgFe-833$Ld*PbmQb zWm49?5#Uqllw{;3oo|{3`o)Z?=0{VTQnzR$7a;e#kUwAp{@qC;4NeUu>t@C_KN+A_@zhRj562oqpDU-K~ zbt-jQqOCSLQL+9Ty)@GD8YB2ciBvk~#v|os%nK5|`X77y+8Fkk`!Uqw{klS5nBToK zhd6la%gQ&9Po#AOJatwMe*Do{4%fQax=G}*HJePk%Zn;^jfnGuQn4woqxRvUAUc(G z%kx8p6&J=z5foU1LF)-B_RCH%NsextSYr!8@^hI++ae5@{mC~UQ$Ktdp&cdl5AxRs z)K76}Y-)QLzw-0cFqMgAL8$K5rJ5i23~me{kFovJn{hYYC-F~j2Burw`M8?kKZ}%K zOFXAKK=EF%XD<)E>`epNn`|sR${MRZ(%Ea~t(p%n7hHRs7cma{)hQQ_H4K`U>crt} zmGX9>*Hk~l)@`??ZAxF;uJ#<7ri+zxKV;kD$u2Q>9e)36y<(5RUCD;3a@-hrCnCHV z%z6oJF9T9cL|z9H9A51;%>0pbj`aB)ecpch;<(F-U-{%4$xAVhcADzDv%fnP|5>C| zVH){IkrGpHU;KI}R`E5Wnh_czwcpGsY5smhGK2H7h(_h%P}j;7{cZq{FOE58@AFHi zaP$=|T!WflkN&eriRe@${%XVd?@mPl=3kP|lYez8P9lT`3vgfe{!ga@fJZ8QqbL7M z()p)T!7PH1bjB4DmMI~Mlz%!Eg~ZLh-#HMHPIK%c(jFy*r1Lb+UJc>R==~wk_P>*K zN+3EFLDk zHgb--UNVK*al4jG6yohV$>7C0yZS*(Hn@eVg4i}z_^ASS(=519xqw#k3PuRBrvlU* zU%iqJ5@`M~6zGa5QX+tm9gj3z9h0)>VOs{DgaJ(KV(>|v9ZH(Fl&4BhUIt>w{!I)A zJ6`^>MM9>X}Xls-r)j0X1`K9w8Fj@?7%nFuleQM7l8giO3bIJC&3DQTn|=%}JwI9vkp zFr?9Dn61M_3Z`XzvqLo*6jGWOgcXqwd&;LJJXOiWE$EyfWe(B7pYiYHI}wuhBecfOvyS?Ku37c-QTfQ`Ux#S)rgP{ zv$RWm0&XStV*~}~y`ms`Et9|6es`S2|3`|%G`@U~NRf~-R(yx}&8dim{z{QZ23Jr; zuV6sLF%yFc45BLpyI-1<|Jk^Tg%D2A;?_p}da*2SD`ZFc

8h!{~fR-}@nWEqbJ6 zZ6sz&hBr?)9xKBv+}`EemjVtvg5P+@mvM64>V@IK*qI+Fe=AdC5r@W6@{JxjnS5v~ zOy9FhqGWlYD(X`BHcM!WgZFySUZXyHW^hZOJqw<=_@+K*`!4(E`1P>&aD84a$rs#X zAFekK_4#O(;_37(B@qv3&wm@+uARfUR=ZGzYe*q`$f=`gWJslgLk=f=^Fl|1)qAvK zyE^Y~Cd)M%E1nJQ>9>>f?RxW@ShDXME%+*Am#YZ6;6ULPC}eOcwCeBR0}DY$QiyPm zWSS}Aa|&bnLF$-#thFdRLB@H>tft0}lznGEQIXwzua+2C1Q>5bBu)(ISZr$p981I+r)c3#>brzrSG6JGtJ;u3%&d$`8BYdg93~Ksckiweg*e z-iML{iC5Ct8U&H>P;&GcZ)qXiL&TNq#wA>$_gg&sL2&*eEm5;l2fTJL#@l2AL$Fm& z%*^B6i@QylT_1UUm&14O7}p_WD~a`n*{gBFddv2;pZchqTkC%DtWxu>+TGHtj-KM#igsUH-52tg{q8lul*Tb29Gn63T_0=V4u*dSrYM#&LPj+A|F@Q1U_)bdV z`PJlm|BJH3j)Ta;tC$9pUtOKA4^KO8qtB`7T3OJi&|Gffg(!cAjkL@E7`o~sdCu@@ z=JP_JCUkn_;f7qhQ?^UOzZbH61##$v%M>RbANw&*uW6$5^h)~fQH0&kzK!}aF|xmX zEAfEIE&0Gw7??I12xkTb&Nh3kv+$`liOrV*M}i_~nKJN5J{)fuEbmtYN(7GI9f_2H z>1G166_myZTyQ1tGRrV6I7z`?3&Jw-7!d*VSqF%Sgg) zsUY}DNS(qfL^gyuD+W(yCg#yhKn_hpqy4f@3&@92|B$*tra~d&^K+F!_6i4T!_BV3 zO$-3zD8OQi$iK@55*$HO5b-=1jl>(xl~uq-iHMmW%H$P^<{62JFd7L_M$lSQ!6Mk6 zhEQ@x9)M6CNEJTO5h!m_-~Aj%#6vd7r`GtVRw9#DB*D=Z^iiZxJC5_HY#SiQ-nQZt zFHDIh!rVsYGKy%-lHb~n@oNk!oplg8v@TO7%sU2$+hWQh8hk34!V18YplPJ?rh3qb zsF!W%(Sx&bk9)(?(Bg-cLdS*T6VP~8yW^1DS(`5e8tvm18BLG$;=`JWH_#JWV1_@V z5@=wto$d(-q=~d)@n@NQ7v70inTa>{?6;KUs?tZWLyERv}88Pz;y^L0iX)c}Z41!=$t9ch2Mz~^8OJqu&%5Z>VB{qs& zPo;Xpd0#9Kw$^u-lK3%82+!C55z|X<#Gcvhh7|RY`?G5c$hTDM{ePT!JO1BXaRIkK zs7PiJms65q!Ei4k^FU-`rKunsy3FJdI&S+vms3Rfprdq_uN_XHy5thbAdghv8&MtQIr9^>h?q zQT|OCmd^v3$Gd0}K&%;pBLM9!`!n4BpP!BvWvt~4uRyFlzk$H<>;oy~t!}wOl z{{a#%Rb=#yS=LCSZMOn1JE$V=5hlFB{vG_p3V0vr9EPV*fuBb4v1@+YQo%Rt(ZoAx zI%|8qG9Vak6(0x>PJDPN{l|5P*!_%(@$3$W zJIP@MLwwtxcn3v)v(XIyYvxsA zsXUL>i6p-!X9mqCc>|LPI{ zJ@fkFg>n2(kNE%p%u5wUhnRU4mZEC^nt6l0+xSB&<6Yv4C|^VBgy&Baya$V@eB3=Bk^-!qAqRJ8vdU6j!C|z>T1Bk# zc!guaO{pH>>vB0NOIYGoDm#m_st8ftxTwG;G8a*=Y7xQ2aPbcE+Tg$7AgWx7 za*%Yb2u{oxhic>la^qw@(>0YkeJTWbkpqM14tbpNl--GcjWu9EXJh+t>-J;$ihy`h;VgU@T!ca zbWoPp%geQ-l1`aIlYhoQg9KaT0vKaPL~hl2ja? zl2rcr@guymA15g>@B3#{De)LPsYF%0aS+xE{VE;K}tXU_kQG#(G0R1zn-^qQqQFb`!|WG z+Qj>GYhDVv+U1M-{WYi1o#45#SLh<89qEo9q7nlSmoj1ONA*aevrkT6weq^Vq_(t8 z@0kg<+u^_TdA+bqn8h#Gl$-3CMDdg4`nB4fX_67-j{a^~xdX2%DQv-d6&>o`%4C*m zPKyZFgwei}1rWs}OpISY2-0$3xknDREn&guE`O++5m~$xTR^WB8el1W8UM+DOK7BZ zI208(G2ee%3Quh)RO5UNyJJVVRe1cF%t?u8$*x9X+ho}?Z=~-1p25gXsd?2!mEV#u zD!k1ojpMQ|F>K$K1Y>?;_!1UWwCzlTF;^||J8SaI){D2%{ayjTTW*&g19s?>aE_!d z!Qx{>!Oya&KVN<4q&~%A5tIE?<=ht;c9y~-h{$}-y>uCl87)4`(+2Jth%Pe}Z@26^ zBk-HYcjv6v*Mk;#M03Uumo0F`+1fQVLY032Ew+ol8&wGVM-rxDt1yRk-kqf@ig2ZVJ;`dmc@dCTnM zM}-+66ys4oQ5I1xQA692yqw!naXX%3q|tqX(e|?(v>uUZEzt-FdY=8u>>bZ5{FwI! zG1a!warm#j%2~Wgfq(5yr9j(AV8jm+Ix4~!bVu_4?N^x~B@iwW4&tx8{#b^Dp3HUy z&!f`c3hvVgy+DI&Q?!()GKj|#W9A5*ptWf%w(O1(y3q;-KkR`*t~9MET_5c4W1nf_ z3c8ZR4H+hOpNso>Kjg+m4Wh9>kH1MHv_G79r<7cSA*VQLB=Yh&1JlI*&|0t(ZL9<$ znzYvy^so0%4`Kx+BH1I}2|{62^1q0HN^ccU0E&~^O>jw&gYh%+I7Ezt+kxq;Rv@SBp} zXH~2H|4(Cg!4-$LWeXSzw@`(1X z_U(ILzwUm2Vvjx6TJxLU+iC?`w(PQIrsq6~Xs=d8?UwHH;ED;iSAS&j`@P_^=%`Ey z)2bV;j)PLy9oC+HfFsk_+G}~|GVRhoA9N=4SUq-*5H}ys&x>DPnQIDsCx{(~D8FEO zd<+{WhoGbVXl^<=-Wv68{-J-Cyan+!Ovp&Pk38H=!ZzB7*+k}BshElMKzE0W_|y>- z8^xUyiBJ5I6ahI^ZXc%wl@D#CSe19lvTpSC%Ep$Yz?N>xiW_-Ppn9;`0j#M9nH6A7 zbnfH=k@sG82f>Zjjv&sUKE`QVfT}Ship(7*mERsPM?YfX*iRwC;uz5Kv#n!%w@BGA z{kfjRF4Nb1spz3oJiN)9;;VV{kGz@VY8=9cw-!BTa;kYms`0F(zuK6YI;cMsMYgRy zy?2n-6qt@BE3ZI1YXU?SFv~f|f{b_tPVdg7P3lHof5#1Xsg)(|QXLf_9d#p?KKFUT zEIhVUxEQ6$M2|DLajPHoK6qBalnzn}*lX^Z^Z2MwcZiO~>mXKtChaimk0C?YemFd> z7;Vk*f!4bKg-vv;?$GxAlVp?;nf3Umd45s+IJ(1wbLuvTBizqs(btfY+vRu1kFhgB zR%TVJ8}Nr5Qr_WdW`#bI+%?~wk#KQ2vb$>aVzz7PtiN&%7HjV)Chs)Zrg&cjkU6H$T(?&k37Ksww_;>tQjERV4So;^9}v)U&!D z)q}}Z9XAOdzdc^II7*_iE|?7F6pq`If4SYKC|b90+4Ec#!(ktq`sJ={7#HHOvSo!W#VCn3esYo z*|ZF!(I(ChB+&eh71F?yQ}mB|>*q{sXOJXqY&N3)BgVuNP{2$X0@CcAwycNW+OgFg zH5j2utfe6&t4zT*PO{dwQB;2`P};>eJx%|$EuT0ZnPB%^Axdh|jMkHXT)=rIWaG)4 zHRcQYF(IsYtEgQt|$P;<2Gkwc>-`x(P#*sG3g zLYeiEBXKT$i6up?j{@W^&6*{ zB{A-#5b@^}iIMUdL(9~h56v@N<{0xiKJS&%XWGA>R0RyN;+BZ{pIVh<1~MU7st3}! zALPg2(e#1QSbU1aESwTy;DMIi#yM&BdfDL6*Qny9@UU$MqP|g#Ka1c>A!R`4s9yMk zLWIah>w=CE%U)45uCr*3_oqbV;^jo)%o3I(@&}TQc{r>MR0v0wwVmc@>(vuk@<5mg z&+a^hpHx+M{YammAWunsr(}v zA_wBuVm2dY6j5En33{#Sv+@HqFql3q!Ly%DqFU1|LmKY( zgj)kdfx}gvY5tS82=M$AEsE{-*>N5ai$*3PB}4}ltt^y}A2zy$C{Z`t_dZNmHJ^E^ z3~6fbq;@)D((r;?1T=UR|NgUi)w%Ym#etqMd4C$*=)V28((}*SP^&l>_JbSlS zv&chr7>feCb}^f~U9Y7N$u!ujEHp(Xr}5ac;~suAud6PYl5>@cd39O;u5mXS@s!iKN5m6_lN%T7 z-50ne1d3mVuZisy#N=JS2V+hl!7p0GSB7HJRJX6g^6Fp{eqr;yfXh%7Gwt-|w>8FL zK^{?5V?VA?p@33xQtpF2L`z_Iv=Ldp1ph#zr z0%gA1Rz(~htDaJUMu{Zyw+74h3EcB=A<&;W!e6^HTwijcd0A<02{9ephpgumtpr%vh(b<|`?DD9M37cpiR`4rKCL()Vesthg#G?xh_(^gJ#DG};<4Z+Nof8TbX*2K$!J2Jq3V7&QEJkvptRTmv#xdF?qQcP-uJed*TpwN+ zrpUdODh;1KDrENOFPyhykLqWk<1BQM=hv!_m=Q96Q!O`OD-oJ*15Z!Td0$`+fKl2R z!+R*ZSEvHl%O7UQ3R!TqU746=!Mp7zMCCy`tPkl~AT4RC6?yP|u!^B~J5!vgMGDv7 zsm(qT=B{I<71(H7V6Q(0Nz zN{KDi7h#1Kn=6_$9O=@R)AjQJudJiNp$c&+Atma)GI}a$KxL6yYIUU$^zti`Oh0g? z1i)sdi$R&$8Cb)|!Kh@zxWMmhr4mW{-VC$6a8*39cCP8`#)zlgzM2DhsE~whh$i^E-{}p7e$*ul+$_}$de_^*trt_1LBloR-V47Cq_th!K z%;uFYT5Xx}$agNCH!FmMHf_YhmB}^b{HMwlZJgF~PU2n7fADMYxv=b?tW+2W1sK{L19>2 zVB`w|@e+#fP(c$|vdVRF$^TiV7AllN8b&2AYA+wNKEqAw9|B4|=Lo5**rZz34sil} zOgsed725JcIH_>!gJ&yDd2V;dS>x*c4oC=unM=mydp?FdMNo*q1x+FYT-4w(sKCTS zlip4ZzBrLM(%!PuzkM6JPb%;g<`=0`K1UZlVp-p+5HVz(UMkO!m@ALc{zPHo{wiIG zuUo%AE9}edXFE=!HySuU*2DeQ#goFAd2rr|y~#PC6WcUB`KL`3cKxR<+dGDIo}r-8 zMk`J`c3-Te>>%}KCpEj*{`xYErwa|stk+RW9d;bQZ+7%K zZ|z6Fao7J&yGj>^8&vFGjB^~r$)H1WQ0`-`cPvp{;lRw6>ref7-lOrvNJ_3a@S*N} zH$aGu98z3UzOClGn?b_HXfZpoIeoFy@$`n>Vq_e`;hZv?!6W!v;RgcuW%H88=5DZ>X>g#}fP&*KG| ztQNnAmlx}9;v0lHKYGirxW(NT&t?W5d(Eu5PP-OAb<()XJeDo49=W&*oS74KD5I_? z^u^PhmmprZVy<2_85^FaS$H&juRO&H7g=$rdy*Ad|I*Vea$Sh2rq1{cXLvv0MnSbx zR%iWWvfb;d#=GOtJKCM|P^3WCy{wCe{P0J8;O%$%_+L(bUkgbf)iG~ot{l{!3k)FC zlQO)J^Y7$O+7?C+)x2IOs-sUzf@@EG(H;+@SDw)U*_Xxi-yQu)|Ys}kKoRaE7-><&xfnWM@-z0*u)Pq=qq~Q%Z==JknDHW?6th> z-H7PJYwmC8=g*nzFWloVwi{po2;`s)6cP{YdLO9o7ueJrAhs8%2MA)x4HWDNyafm8 zng$(H2k8w2>1+iFnfXW(_%jfAQilXY7Y2Mo4E`V;0LS2Gk{5)(>92qs5XulBO&_cc zkqD;u4H7U3f$FQ?B_YW}A*;}I*{W5)Yxc=+!E#ZK_V(ZfypHM!z4+y_kwbIr01}zaj=}SAX+gX++?!CMIt{} z9FP+58Ym+Ph7Czgky3+wVNcebj;E&pPyzv87HMeCFu6h#v7J(FaFPJcphr+zEF*x* z5lG|sr*ponHLYSe4dR$u>6TU-n`(6^mo@B2Mwuq#B-$W{)yYWANf=P*l8)z!#l>&* zEtHy*Fe7yV#88<2cjx?UXp)XZ2E{GN&KVXc4WxmQ0c^r1LhA6ypjGY0S!60%WNfh5 z-2e-5IPKIdvTnc;Kk(!p^CC5CfGd5t)ER#YmUP1i<^dBpfP2G1Z!?t%2b(jrNe^*^ zjW)>vHo?%t(W{Ga5H`UOUw|nEBD^@9E*p?1Ci_k|OWrZ-%|_0l zTh@Im46_~_pa-bslfB6ZREg2nk`O2Dh7*p-gO|@?2I8jV!ACjakf557Faa6eVGYhm z=d6Nx?o{dR} z4J9fU{Sb>RUQA}zgczLz1->8m@e-4oP6P826IwcrpDgjJ`W~~_89B_v#ES!7(vcEj z3JDIDh=Br-!3ga2!AO0;WOq!Z7lS*WM7CuD#`~zNnig>sgP0v*r*hz@x(T$);E!Tp z=sp6}V!3-MfQ~un-A?%QlQ28o2>Ed^^kV7nIZ!}ushh?s?foDGDqKewfGZIG61U7T zrW6AR-xr2N1B8RXlL(W|^6V6-XmkJt7vfk3QFk6Kej$8+RHTgBBan6`J&z zQ&QoZxT%_Qikq_g;9?%((&E1? z7x~zt+}jc(*m^6`nl##+oZiY_-kNc&m-*N_fY6rrl{!waE#_F!?n7G{n#)aqOksN) z5?p)TvZc0*ti=>xSVDVSdwa)nd)H%oH*rU=U`M}xN1qK{H8C~#L&w;1$B)O3N#f3F zg^oM_=BS2_ne@)3_Rf{%&MEy)d6mt~26q7soQk z`ps24U6<+K9@@Vx>VH#Pt~U$#_9WO1Z_wR!{7okubPWqeIj>i~s8MkN0z5hp4SMj# zI+0Hri9rg+Ts`fAEbj2V6k{^f9zAp(y`AYjR^=d~-GuucAjudf*;CIq{ayx-zDb)t zo|6tHk4lDI0ivC5H>z$ys0pgy&zI3pz|twq(m+zHq0mTA-wnp<0V1Wp-BI`^OVZ0# z(Qi1`Z*1EHG1e`b;$_qaa@xumbqv@lvdEw}d*BBlB@EcF41A90)vp+MDA6Hsj-rh3 zxZ%Ze$ryAs7#a}l4MFb+fGDzaI*zD|B4BYsfZTEDAQlrb*{L1)l`uI9{T={T(N9fyz zf$|o5(r(9(bLIhy{_zu%iHYMM=I|329uwacCO()=+^tN6*>*-!0(TO>XOthrOF33tNbK zUch7>h*p^hRa&GSUqEqMe9Tx(`!$^r3CvoZj3-@!vs+B@Tq;?eD0^P4SY5h(S}GF$ zQP;V+nGPxxp33oDek;6$RJ%+-y4=61Jhp5nx%zGm-(!tCVeN=?Y~vkp%X4++c`b-^9iDxC z<=y&C<;cmn`0468+|xQS&IX|C8@$NiljlZY#s;F<2D<%^J7M5M{|L!xdPLTbkvOijVr3uFe-TM;>Ye{zU;VVZZ8-kvsdj58En^n+&-y*WqXYo z)V=TXZm$O7xi?!0D+IIe7~q%PI_o~3_&)L4VU~Sw zdKbTs;n9)J;W+DInf*|46@Tf=QRUdtcI070*FfzlZ++G=zwPl==Wdr*U;7$wr^v~; z{)y4YlOeC6?ymLTwUguL6FjnC)34TZ3_*FXwhFv{VP*Zosr^N}_G@kJ$L!kHoZ%^) z*QrF-=~2|i$?4AN%V{ggZ@sAFtEllC!~M&w-+Ql4u_;e^luut?#vetFo(<2az0Ov$ z&QQI(MoziMUe3aB&V@P8alTEWi=Mv9K3}yt=TJN&GdhNO0|G$ysYEZ<+t1sgyit>{agH`gblAk8<&+UwV#%4^e!YcuZ-^NHj4**8cXHv?-o_7m;O z)vSuY&p&0ex@Mody%}{Dy(Pi89eug=F*^5qa~@!HM5Ebd(rGi-QN$PT@U@)C#Bv`HQ7%M6JwPcSUsXwiQn$)ZD_u}J@@QG zQHDss>b!eS@_v?zd&U$UuV8Q8sXl`&gFa`{K1FQ&s&ON16TBp8y>-%n0T z`EQTRW%G0wn!tj3WY0~kObaC|F>cT@wH3Z<6R1blF6316VQ`cbgk4bPcCqgru9T3k3Ho(&1(z^U4-(_pFXmD3x{N7M`QR>vnNDQY(j=i$OAm$(MX?1X9I z!jy)Kk&vQ49vP`37StmHmnc9zvYEy=-!*1NIH4Zdc;~di#~-Ad4VTJ7s&VnkBFtm; zEzviRA62l*PcA1RC@7jPs(iQLtEnRbS|t@Z=AR>~vYglBYRZDwAax~4sqvX+hQwG6 zjcAUaU{!r3R?P~nz)xCT28}=2^%glfwB;PGHQ5at%irjzl|EcAkcE0GYkgQ&+*oXl zNxUgFFK_%)rj~djWY>Ouqwm;<>uTUU%6)6#GNbAG?(=dZwvhR1qN}01!t-hNm*bzV zMqZECw?;m2cwdbDpbo5Y0Jhc_lOW>HcP6H}zZi@{pk-?BVi>x;64!X?c{MX?guAw8gjm=E0_>5AUmO-6)UX!j8E>2u0Pz=jD|qjH6S#wp&el zivM+ku=Bre5Y)>ezL@^A#;oYi7WRlsu5a@{Ys^ALkgP2&V+W**QFlQ*y%!s50YDl< z;r}q)#uuiIMg0~A{%$mBcI21f=F>Le^NW|aU$vZ28T#t<=Mu4n?MEHL{rBaP>ijg{ zW<&rFw8jj&K|q)rPr1&XO(^DKf*Y2h^%*NsFE_MvgMQUeKoI{6hpM=cuk>Y$wv)em zCL7U2%$5G6Zz$recUVPetR#dX}$8g({p6mSQ2En+oNBF$%R5jl zCE{_E(vGz58MszNHOA9^+F_QeGif1`Dokg#VS)FhP&CqGggZYIC!ye@JTf$+Y<`Sy zm1YVk7t6p08G1e?Q+6 zo2Rqh?Wps6-5=;5+CH2#|9<&%iTF*1S5gr2%uu8&`0}LQ;thfEn1ldi#Src*+z=Eu zJP=E|5fMXtUqu+}1BaN9<rNytCI*(!)wL++;7m0w(`K?$NP0rMe;S28`LS_ zjoARoT9prBbGm#dxAF;khrWb|jU(N_a2Tvv@J>egvUxaCRUD3!_(gV*J%Co)(ZN84 z#KxHeU%Z>wryUs!ld5`5iX|FdK3@SEp?}p|t}U9d9VWXVGmboL5F~_L;(U-2q9ZSz zrbZ(JWTaG+S9AQJk;=#}o1-Cbn~11Kl8$^ZB3b7QZtTEkaj?oZ5Qt5tyeMq`j53DM z7M=3_DT=KVG z|1jLN>biD6)t>?B=#VutZ~(8RuA`KD)HIuB6D1vl%cJ)m65CBVt5sKzb)E#2x|?XL zRyWJFk&hMom^5k#ZI6{m^aX~30&9m0SX5aRw1=0#^^Ci=M#B2olVJ$8E2_5IZ2F2b zry336w5&=7Xn3;_Dw#$D6IM2>N8DvHq-L2Z);Dg)xa&knEz+P<*3f0#Z9Jq_Db?4^ zsiV02SV(Q)*w=`qUvZA>5ZfiEP7zvdaDG=Ic1VD#;fM8cF3S)*#Z?{P7Zq@Bp($Ch z*lM^#L7c}t#BZWg4siF(I1qEhZb48D0EGkx?mc3UfT|;a5FH1}2(gzx))AKR2^&ox zv5#+x4VJ$H8&eyxpBLl=BUgcqt&aGeN0l8$S4d!?N~?)8mtEPwmVjU$JOGtv^K3Kp zNk+j#Jey7?9*Wo$mzcv<)Xt8PB-jiy;Nbw13&o5GQRWHwQU9qr#TFs6uWq_@#&{7a z?VV04Q`%&IA?rk%|PX^Ub4?1>O z+Vu!?-}SE)EZ<@2S0l{R#d1G<5v~MVXwSc}C?FYK!GV7?XBF{=hIQJUQ7 zWGG=-#$zsPaee+$S&8MCFTZj~>ncCC%IPwwy~<$XD$~GAX7tUb?E6mIg+0j<*%g~h$X8zluAI0+@L!UC3#i90! zxZ2e_l#Y4@i0q8u_0^R0Dz*APO;nu7NcsEK4bAAvdH%R--tR(v@96~>nhk$7*c6$HxJBA|M>kH6~LCWt<9-0qBRE0?PA2DPjvCHT6vfFhXa_tdjtV z4*`1GATf zZN(C7ARcZ)$Q7Lc`jpGy1Wt%|0nuX<>EeKF#rzxJY|brCN$Y~k`U1rfka8QX>O;XPXQ9T32$XR< zh%eNW7cp2WT)!+dUJ^DHC_1$&`k_%|i7IT}Rx~tIWX~kzNL1X-S?qEo_?f5}u_cg{ zBiv07ER{q(l*AB~#tD=r=#@r>rMQmBhoBaraEB60 zL}U@+rMnk${6wa#gSBQa(=H39jDz*~5-y!vmTFCD$PivHM1IZXaEJB)yVH5Q}S+`L%wQ;4zc1LYC+^Z57FDof>ZC-)fyiY2%}68<%V2HEkOUZR5#n8y#-r-f9~` zY3HJ9AC_z9G;J>lZRf~q|25eDW~+T0spB0%@<<6Q^abvO5@E#^X{Gqy=G)bBH$iQ8AZia0We;&&_h*Y9 zBEKH{;vRC@9>UEYg5Dl0`d(^;UeHrFgAj<$mXb#3GphmRo0CuMij8{R$TSAisXj;(lOnKka@$Cg8i+7?B2v7{8FhaapeoQKO{6 zcN5fs1inaT20U#=v4BvlY#jha3?;?cfL%xMPgFdM3j5h5JRi$)7nT7Kd+6X%uNhpq zjOri^e6o3Y$+wX~d`K?rq8hL42x)lQP*?^?A5_qW+z=muHD)s8n?9`R1bTA_`shUT ziNB}YUE7oz2sQBBu7=1@hEZzqQycI<#^RT7Z<9>oCb`0oD?WHk7AH=k zu1>ysH?eUyi6uORqeL=dFvSa+BIulg0&WgHxz+nqy1kQBtX8y2G&G*mwZ+pckwtK; z)1`YUmN}FZDsxbKqBz{ z1Bv`UbbC8wajMBwYX%0gLT#{AjY1Ow6S9GPezl}-Jw!H6l&Rj{VSvS zCG>{6>?9w{_vxi{^l#lhG0U>#_xenzDngUe-;)vCuum;_6yz9~)YLmOz%Z$40#ysB zZr^AV8;v<2ve0_HG5^OU@tm>&uS34T$SkTHIIclm5>bzpeOv#DO}F*YhLm z`M>G*ZHZ#g+sdJJ4J4+2)9v+?0-?HnSK+1%x%|(~5SmkXB3TSnm%K2D;c=0fSH3rQ{FgrAvDVAheU?Ye+yAgv*J82w8bdRhWsYqtEn}{qBqTUp%!WjB&K1feiLx#xTHsmr&%XZ}kExNk?3#5FPj|nnO(RG)NEiSn?7Mk2o^!lQ-b&~^i#Qy zg~)QDf#Qe2F)X#ADl0?S`La^sinU=n1zLE`eEqK*qr;6)ObG`0;+f+>5fnoCM7x@l zI620S^a~q{2Q-wthKSxaieZGq*fIl^b~c=fX;NaBuWvoXD4hGz_%C#l3gsB!6BpsA z6H7q&iWhN;eFb!HINBv_r$cy_x9NSG`BbAWT;aKP83Rj3G#_dbt7a)#-dpwQPG7hb zcI@JPELC{7RF_QuM9U_}FlORjlQNJ&!4a?9{edGsVM3^oGk>w&N=@B$u7mEhu2ku# zQ+;~LBORY8{ST+lRT+C@Bmy&Tt)J@DoliY1wznAH!q3G=Ty+$0T$Dm4y3Xsf-5%-1 zj_D?SyDD=KGv7u*T~wndB&&N_T3=r+s9QeM2`Li&arT zSBNjK?p~Exhokq}^RUL0X-0dhp(v@1QTe`LCbO=zSfmq6#oxUk7poC0EqoLbSElib zy|I+Qok@e0=4Yu|Ntsq6P^%s_x6=8OgOR4W4(Y+qTD+19b5fw*3{g&FU81AIb*BDy z8%5ixebwjhn1+|u*_{`OB3{qs9sp*YJ781IDL;$pV*bK+rr_i#41Ff* zxRsO1mhAf^4efMwgR$`18JT&Z=0VQbbB5yOKT|>qZ=4V$JQ!F^bTeh(0lTG32f<_v z)Ptox_W?NrSmVV;p`Z14J_ZDsUpAzP4!Xl+J$67Oi;!^i_ox`68Xqub%@H6W@8jME z=(Die+ex+bkpS;8IWNW01LlmpE`)Z`MPLy!H(#TK<@hGhd>OhPeVh<6vWk6Xa$q=u-|vV(;aLSZlst;EdpjS=;w9Ntz(B!KtB_&||Mv5D+2CiK0@ zZ&r^Y=x)1Oj;o7-_XF|-f>NwbV-39|B?$`O_mVKNw4#h5E|j4G(f(c_jwJ;mz^b1# zND*6437&gQIplkBk?%_+Fa^oXRgNb`h#6s;>K?+>D#=4O${4mn$>cEfQm}Ut=m8&X zIfhzv-*B0Fb{}8a&+k|&;Np$v`AG)Ta{AZYej)Ynp`(q!tTy#^>HDH1t!DJOdV8^| zV7p^OQS^m)$?hGkk0(1Mza%H4Mr#}*f7yRSTiG^DZk<_`bj?m%y{t;=yd-7yf(X*m zV}0rQO=BHctguyU^(ETW{=CZHW4n^-ewexNB1r~ykJAAM`6lCX7VcS&Uk+H~=_OJ; zF>R~FD^eUnRz*MMGwHUbOSROu&`|EFLwW>IyblZ zTLnq{MVg_>V>P+JA89iqF@;=f<`W%Hfhq>%tl8mtzIpcPAWr1LDevVveCnAqP_%aG zjd$wgw-=>8Z{=(6(#OkenwMMe57a(FQa*NAK8Oq)Cw6>rWgn0$OOKG?7ld% z?nJOebrxjAEmYh1zIb1J!{R~Y41T|-1g5UZ;EesadO_W6el-5zU=XdW{}G6f zH^l!S*I%fFT;$IG7UY*9?hn@SgkC`j`l2stlE4mlWwDwxGGm@XukFF)9zCm81- zn0Lng2psGr8)AW==d<7zqN5iiOA!juQ9YXtv8D|LT!f%lh9ZW9!k2^shC(6xp-)I* z5Ao!EgAuO#5iUrPPV|xXGLa@0k%qyM2E~!OgOOVMk?KfMD)dpxGEoW^ zQF6giGR09+gHe+EQQZj934GC4I?*?Oxg>9BqnmbV9`2%H2x9>J(3aO2L_$`Crcfj+ z@~&HQhB=>~)aGbzWDH`aNH=6OcrjS5WGfbdnNzXzuyNS@ap-b!(yH)FuGm!kKxV7B zV=8P`H&Dxd93?xOIafSg9VniQ(E{*E7^5{lxf%3~m{8f{?BD-^0G!}b>mOwTBD{)~ zZzWPBOw`a#)N)JIxsOXtNHkbXG?WY7%=a)#B^6~PHA^aj@@Ml3e^z(>EGXYGznSxSj2x&D?b^&m4V*kLl^=W`P^dA(RO>}FKQ+3<@9KEwE4FpgV$dR&YH~%x=b>|v6kBH9D8ymdp#%Toxzm9 zsTC+MC>c(s)YkBaTKQ8_)p-1e7tB&>Z~C?TpS<9V@~i)w7rZp2_)}5^u*?gz!Fcu+|#J+DNDS>wXri zx0;|=eu)(RIHw5s;&%TWmqZ^cx__cc(ja7|-zZ3WA+*d6Esgz+V5?NesI$F`3KgC~|=W}k7 zvDSxPb9x3V)nJpN!_;_4&fzpGo=NFmM%%AD%tV*H)(!OIT(hC~{HO)WV8t6^5l=2zo_dHF9( zd;4QntvIi8(yZrCtw%XXhfJq!uCb%12B&#E6_rv>F+NS>+IPc$ zrL*fnrmr~bMdv*^>jTQLocH5s+n#?Xu&6j6AaOl8AEXFoxfr79b8#GIC^m3tW@8#y=cYG@m{k*9^XXJo%d!+^>2(?fCPJ z??!s_^oJLGIv=9{Z+OAK7MvTwn6cwqm*$sK^?>(x_YGyIa%1Z5Zwh%GR2&EmE#vbO zP*|uUr2VchhEiM%PH1v749~9tCNc*Ofl56Q!{heufAfNaazDy+>oX1%iv{L~e|&Dc z%sh)I7TTAaa9_I2y1UW_EQV305-dNPM|A!FD5)ybyZ)~W&bhj@>k3Pya=rPHxu&-3O2^1@gMIn=E_v=)o0@( zTjEOS4goj=`DhEQbeD#Mu2f48IOfGE_%*vV#c*q=RDW5!{#!{^1J`FY)kN*Yu<&UK z_S@Fkenb~tfrUf8+kY*oQYmEpGN`b2k=FdaO}N^bA4bUrY=JM)3at@%m(U literal 0 HcmV?d00001 diff --git a/docs/tutorials/images/preferential-optimization/system-architecture.png b/docs/tutorials/images/preferential-optimization/system-architecture.png new file mode 100644 index 0000000000000000000000000000000000000000..73dfd8d659ca45493e5d31ddc47301f2657caca3 GIT binary patch literal 82341 zcmeFZg;$$P*9RIrcz_mYkt8^ULXhGv1&X$`#fw{UcXy|_OR);Yi@UqFxI=LX?%}3K z-uHap`R*TZ*ScY4J@f3DJ+fzKX7Aa*AxuR{1|R1!4gdhamy?xx1puHy000zbASRMx zvHUy?`C@1;DXAhSDG5=rw=prdGzI{eJ{aoht;(@7_3G>E>Gck=Fyq*}yb2ACe5L0z z+}hXL2It+4Chbx*IZc-A9;8VWOF1Lz47u^=*yeC=wXK zdYH7B4iFv#ppk8NZV!5KPbxG!*EcsVHxwEV&4dQzqJNqrVtUmY+YyUnL zFoboDH))mIkQSKkA&7>yTXlag8o~uDv*3i?HX-is5BKly?{&IcTRkKER$6%h@?s{Y zqC9`^)W1ukL2}UkPNOg*^8n(il5%p$r>dd7 zv9Yy-nT=zp4A&HrfNd+Q;Q#=T(*ORV$i1S20|2Pg=C9Qq)fE*43~j8~^o?u`jM-eR zY=74S5ONhjCasJe^&zfSmevjeuEI2bln_9sf0Nm1Ab%8bv=F9IS5$#W+SnUIc-UZU zFd7jY2m~TzZ)76yO6vJv?8rA^8Z$>nTLE@<7Z(>c7cMp%dsB7}etv#-7$-X?=Tl^f zrw(q`j{2@otsQ9ptmN-{q>LR5?agf+&26k9zw6aEuyJw}rlI-W(0@LE_S4wa{J$+( zJNz{)$gYWS z^6>Bp{n5_9vi{rCKUv>67~4zQSRpwbMgBWke=+|v^M5n`(Wb_K+vMlx{AZK@$oU87 z?=c9xGIp@BbowpBH`eBkB1pFXi~PU2H2%vb!okh`7t=qf|C>Yo|K#{5^?!3H+nXch zp#QsTA{>9U@K4%b<%QUP&;37i;ZH^TgNoEN5gZ}*|7fiUj!&A`AOIi+kdqRB?TWIO ziRHWSCh>lxN}(^`Wn2<9@S|+G?}Nk^3@M%6C+8JACE*7y;cwq#i%tWI`Cgv5cF6xw10O>4;TpG)Kig>_*WJPRkrHiL~!g&5g zL`4f0147U+o#-C?4|)GC2=$es_!rR+MG`3tlmG!@oPS9iISu&#BL4r=@ISTb|3lO8 ziMlL7K$aWqA;B$`+jZ4BoT|5uJN?hH1)n}u2wZ)4M1iwF@nw)g*tu*ZyHS>wKfQ;$_ zjYL%r?$k6iG-_RH5dD&!TAD!D`!%nJcdx5UkLT+}eWG{A{KRCuFzq;*yf;FJHc(I=o~9ABh?Az8*i}0XVF9Jv2TMd9vg4>4)P| zLclp{o$m`>@$8lM(3^Lw zy#jY!D*%oN$zYeZgp_=-BC}~}X{d(0(+^3vu~4a?%V>)7^73i>$2{rAtO7V>*1y8U zs!RRba!UPEDS0f_#Q{%8;-pKkP*F^kw%IQ^wkAwRbzQSLKv$vM*90hsBo$7$?2-%D zr9V>Znk8uso-m@D6-62%P7t9SPg(eq4N|YA04sSI=-zl6fR}1&$(W~?X7c5;w91*# zGM{^I3u|koNHZ%dE7-$KlsAP3GtJ@k&o7HhOO?Ieq>?gM2!{ekz0VI8(izyuPbwb& z5FdO?w}6JOed+5Xo)Z`%o|7-)#ZZSe7BnTMHwyB-Sgf(p-HBP&Z4`a-)7i5VHI#76 zu@h$P*-0xC9ZBdinx;qRnGYNdb~#+s(R-5eGBkCFR@w>P;h`&iH!dfm`|Q$*^R>O0 z56a2x=x8jaSGE`BlV22He%Y>|Au!qkz)rvu(g(l<7F30BSX(eC^XG7qrdzZu;0R^G z)Q@R?Yb;OoksH};%&V^ly~@g<={g6~G^nW<4m6*!H!(}slsg4#@G)>Vg~PDFFh)Xs zClvEl*8w?!Q5scZ-RlMC{nF)0CE%q1R<2bL5&++b2D}&)}>&#wWDiDqOX2I)5JkQKdxUidDP3i z=ZSB?L6{|Z;5$pxb~&~0QYl%%v`?qdyItT?jxv$tAKrNV5ziBxI~Ie|czZ{@3%#ZJa14$WdKzL5ldUl;lYq2m;B{5KDxxg%7Ycuiwnzv0Y z67=|Ey5fs+MIpquYj@0C>OE908JV{2*xLd&i;a$%B3|`6_I>o#dTsu%NiURNf@&*T z_5?BW4}I=$&J#;L={W{2FD_7MxW ztbwhX6RyW;iN#d3Bw%JLo{iYSahD`$OCQt%YD~#-FqwIu8NqR$5Ta+yz*CiUwKZR4 z+PcIFo&8XauiJ<x4>3xXiWB$4D-f?*$~Nor2CP8z3N=v!x*NEQEA_5r50|_OSt4E*`Wni!@4j} z+i^@X%ECSab0C>x8SaDaO(J+349>@i-l%heVl!9$bmmP`{MGP>#O7N`51$}>R#DTB z1N%3&h2Y#3Ic4`(Us8Ool6}g3?k{|5Y`jm>&vOaAdcE&&mdL6EDR?Z)x+2IJ<3As* z^)T-D<;%s7M;>+%idt5hjwqSvM36pxVW;1Z@lB^;GKozmZB_tvIE*pNEHI>%Si*Ov z+)#$tUA^~yol%rO^?Tb7D7RRQ&~fNft!mbxdc{nEpGqQE>j}K~S#Q5-CLZ^Yb6U+- z2qya1%b1R)NxO1?0pBc@1aNlol9bW%jJjK_5@TsghkCoproVAsrKSs zVf6UUApTEIt}MJo?plK#Chc-?7U5>h#psIdi>!WlOfv3s|8oM>51Cg_J7}n!Gzz0= zJZ+se%I3x3hmkzMmsx%BjB`jWXr}X>cb>DVHtHi5+^?G0o3$wYp4_-jugtW1cd?Y+ z9|PvWOMPaMBz#df@}0M7Tk(A1(0i(+e&>nD&#&*!0&QKEyj;SQs|oLJH$`j6n&Ckh zKBn1T)ptwxciB+0S2r{GwrWDgG;+Z zYwgVW1(sOTqT>7L)hFj_8E=m~Vn!y8O>gVcH?epFCCN!r@Jt>ap(o!X z4)1vmh(xlE6N>UZ+V3uVM3q)b>Ne6O^(55wsd{&Qe%-%2^zjK_YeDR`#ARz)*U@8x z54|rMhHA>YjrM9$)2>72!&ZsURtN(03v&JhoABXYl(6J3w+KnF(p7r6fpt{^Laa-R-7T(fn@bfU4(y z%rOz;+4P(F^)*{>Otcff{$#^|<&|4p^JLUBSpc--LGYj|2#nt3TcDCd+lTf`Z#hT% zo%nnMh1xCL!R0YeLKn^M(n6};5$tlLu#0Q^%41Qy>u9-%Pn*ZXs1pVUmNH#*>UQ$;KI>li`VW8~)f$GALb2xc#T{o)g)HF_EUG z2}CflKEX4))y~&+vkd-N4KV%sPMWi`-6=8#Xe$8fKAgcH2Q#iMAW>$9Jw-aosesnP?Xflm&I9DzO~!Wa2vKg^j)?wGuaE@lcc zrt;`*7Nb9tpo2;Ra#Lw^z_jL!h>j>1Vfy}GEm^_)g!k{VRxNA>rZlLTx$ItrXg7IR zBJ60vI#+$y;NBa*VKOh#{Vcnj*R%4a3T0dB%z=cmXd32siAlUB0sxcsl(UY=Fh9b% zd%?rTiN&)L7kVzVV_&Lgf_?MeH~W3f*EaX(Hc@TKgea;ZhhB#s=c{Bbe8QD;cP9U% zrbHQGCf@2@w^}k)N|bv3Vwu2W;#t+b9#1BY_XqdiiQb=xp5G&u?&bBHFT2a7`q6R}iPxY*kwM zs!vt>2bI4Dl8bEYHU?A8Gf5cqg@@Fbzq7cec^o$JCi>aCFRDxOYwZp%KXun^j-Ji! z@gq(8Zqh^TVY!n$p9%Ah=_d1m3#NqCd&nFSSVJ@g`+4anz=<9Ss`8S)whi?4% z>wPikaqG@UU1VhK@{m5S>xpcok;ddYiD>xxp(tWa)V?v9S^D5|Ek@aT#0PQWkbr&N z#qRwwRHps4Qh=v>(vY)z=}dRK$4qR|jYg%(FXzL>%*B&UWuI%+HbV2?MiaZx03PjC z{AkiVcZUY>BIf=wrcY5q)jscOBCgxqD4v~l-zq&nw%WEJQxRtZ?xa_$@{WwA->I-U zs9()kat zD}{0)>xF*&b#j-oBd5%1KVpJV!i*KXPl3l-DtIZ+1Oe8j=IDEvW1OS!|dTF}A0-q%9hLkX^RrVGn(dXL5c6@j96Rap zo473W_lU~2+`+I42vr3dUc(@SInVC0o3~m2OIZ~xj)tb`>k-{I6v2+N2|O;h$K?1o zMpc+!*4x{86dPxcS07JK?1o1i7Wh{3V(j|-l2bBjsyW3P_y#&yNhs$`caHd4$O zA@|O0ej@0r4O6YAl$^kF@{DG*2{uI@I$q_u7L9A^$)qvH%j-Da=A)=>I&%}zTvO4T z8S}TqiK4p-kZYoAEa&0thL)R#D)DX&)66&Om(k8>C#8hm@PN-tMjLUNrPH!#-|!eT zOUmVlk}k9DZ5-$cY)UI-nnsMq{1miwEQ?Kv9xgOD=M%1b8~my^xt)!|@haZhrSdj|9B zAh^9=sO+jvYW`aoT8PqB59_5?gAx(jJ_Gtz2O z*77s>h{2P#FDvnILob1_9WPS#lW@tp`g5O({VWe}@q!rNsW)|#WLG9M0E3F0loX@o zn}F#RoYk8Go17N?3bc0Zc$*kM@=B&*g0{VrY{YiRAF9Bp|jOiHa>mu%hpl~+!vOZ({iRX8mRdbnj3E0Ge&pnEELY z3o3G3CXRjtGsjt4jTAn-vfdU1Ii^7L{QbEFXtU|`UZE}m3_hZZ1ly#1>qvElqv_8? z`iJ564>&Kchd90eLAc$aHFnP-CvmM{bWDwk2Haa7ST@#!uZY90{O1X8aqaZ$htZEG zvJt1;eFGkwR;~D~>K?(6ATb~*?0}8Ww+oF>C{{sl8FIYQ)#FIZ={`wB zpo*Zl$5%rP?klA2Zg#(>zo10f;t5zBixqJ@@w{5RM|bHHrU3h+qZ0tN7yJ6OS?c8J zGfaf?U$FrpuLO%+e%sxD4}iuW5$81U%K-& z-<^zziel^4N&?@B5C!Cj>#hkAy50^g-6|e$`lR3uD}CYAS4d&41J4W0Qd8c}ICj*K z%ecnLsfGJjMr~z7u9b!r3~p@4kK{-fKVZJ4Iki)Qhn=>JihY+<9+)_>=3kHaIvoUiDKHDK`Y&Ej))c~uYwQfDaE^NCS!Qb1 z4z4C6^_Bdk!b2S4J%Y01F7Dmu31^H-oNKZN(wLvR_eOIz3X`PlqWPk!@B9IFRw~h?jpr&!wk#J# ziV~>1e6&Z*-n=?c$MJNmz;pz+dTIJS31KBq27ar;r^DKg$R`%!DVgE}9ed1`LVByl zvP9k4$PgYME2<}N=G|r1ZTjjC;__|X1`{0s{?T2d^%^3KrKTDFFB#WD!QClN;h3-v&0a@>rUel6lB;XFU4f<7c?5gR+; z;vI){VZEh=Gp*ylYoCg6o43C6dyYbl7WLEe7L<_WhRuz>h5Xl z=_O76bZa8dw<`JEw-s|-@S(s{5oaZO=LXf|K^b2_5I-93($=&r85i!cw03OKHdWLd zU%en$=n6``H-1$HJImq9zrBo4qU-7cACvu#eKtw@9)7OWwTnE6X-oBJu|G_3a=2Kv zMa<9KSW;56tKI&duchsy+H3=#89t##-AesR#Ht90$qC)81B53`5($dr?o{9QM^X5K z`!GdJ5WEsm?(hp8{q=+j6$s?fmPqQ)qs84;Hz{6AEU(I_bgyMLML98I%Zg^>?>W?suK%aX1m!n4@(nT#h@ zb)?Ay**aSROZp!tzdwZw!&N|O_@P2;J!F?1f>0rS>Gp#FIQQ~M{Rds-h7uM+lTp;w z6mRhfDRPP>@=tj%YUfY7l7PM+Ffdgn0Jk{SdYe`KHfSzHcwr3#N}>0xrJ`Q9r_D3 z_Oc1a1?6sU218v;LB729hc%pAYy{Tm45;q|M?y7 z(zuPkP@1*9`xw=}*{>E_MUp~3l=kdvYjW9V^hVSq$`oSbzDH|&yxW5uZYLazE(J8j z7O?OuikRfu6Yte z9ZjVQ9)^GBqxMN!{?J&V9)SV(&Y?NT3hwElVI#G+^G!V$WLNsQmqjH?+FuTXk>)1L zJRGMkIAH6O-dZfA>{~T;VQ*fIvA?RB)TlMMSlwK8ZMfOnc6q3)b||!zKb2WqpI-Xv zXwx`GdaK|SZOmrJ4H`!@T0L;VVcAMoQ21Asj`J$}GRJL^#%0E&Lj6TXVDM>lKx7x3 z^mZj%dQi#WfY6Qco497|{*is+mGUeg#Dla#60yjeqR)DP7D~UhZlwQV0^Qwprz}k8Di6K?zVjC zTxdAj7K2CW?~B>R{WbUr)thF4OSX$rQ6zLvNT|>uL>(X}8m?1bQ<=q}Ncrg`wU3sX zQGtMtmGS1i$(}n!OV3p1*P^z>A?}oC>(m~mJm-2FKkkgyc6*=RG;1F;QAWn3oPLCw z7CypvMII+1 zD>vdga95h<1DVrJ2dhFOYBlD3&50iqaTCs0K94^>R{UtEhmX14v3B{i$KKDylpvQ6BT+)<(0f?yv;%E4ZgpwqDo>xe zA8ZXVryYFCfV20dz4DNtgUzXVVo|O6II-#=f-k&9BdgLn%euO-iJh*ox=8^ILE+R( zj^@o1<2|w*mC3zr5;nvG<2sVm;E)`Rv=P9jngD{=8~V|XRtS$hJ?e$XLY}he;!GnW z5A0CiQ`+99$l05Kwg+Op^))Ba;cbGmpvYWYNXCGlxk^%Frtw0>MTkfC*?^Y}qM_f^ z?(#sybZw7mXw|g>5wY|zWb@Inx8IF{Xst2xjR>;bn%mZ%*QCw4$gb$`Hze;NB6?jj z996a$h55@%aG4nhG$H~Za}$0Kt^~WvK#Z154+W1Yto>TY0s#S!1~Itef#AVo{i&g1 zsK0FWQlGb=wN@mo%bnX~sEjcrg~&+%)8XqLWH{ARPjyIvUaFqz1EII~P+^1**cUrS zPf~NXK~(yee#Mc&ZX~>ofeI+y>GJc9+9Rx74*3CA2+!Ccw*g^{4lgPYA=Es4EGE!( zSAHWcMiYbBb35j{<1g`R)uR!+DNsBfAl^@M@wADpF`VLbCBI0<-_^IJCAc+O5MsMe zx)Zjm_h~;~ToCAIToGDoJPfVaHd!L!q3AwQ*$aUm)Z_^*gIz}Cs}&k)k%m<51*hBM zk&LUi()=2)e63rSpvRJircT$8;PL8J|MgIz=#N85z;=Fc%p~R*HsuN>(p2;E^~FW; zrs4eJu0YR+3q(Qxc-U42a1i}*4VyH>ff*iC*~0eq^_dHa_hv(Qf><`xQxwEfY?en{ z+B(qDNN{;`QwoBTte>w83Ywr)kSJ{sbcW;v=mjhXTrZv|XK}C`FYJ@c_VLaV9^*hE z`t1OHL@Q~!rlDzyPAsDWJ!G&7hTrBn$lC}ZOc8`DPzzJtt|{>oX^y_|N82QGz2cZ^ zr6R$ZnJp6XKmftWFewkJCrjoL`7oTbyp;}PNka|691+r&^p%$P-n)UKFwaqNpr&Zy zWj&(ga`wZLCH2wLOKKtNMW=Pq-6X{|*^{iOfs;`UO~hhpmp)-K86j_e=}t<6blmZH zdrxjXut^INSd>SMGAr^RVszl^b=T|UA2g1~XnjekpGPPoXbSLq`y!tRSNcwDJ7nKz z+ph)cP?;niV0S2^X$V{;dsykFPB_Q)Wp*X8Ds_3C@6GUY>~B-^eqM5zeZ?j0A*4-D zmmlvZmwdj@$Ygrg`&=+HYuCDSsL6T5t=+&Cn}0jsVA0|t#gVvd@B>c7Yq=1T55jCS zo@58~uk%$qzT>L24c%Q8k_B$A8_la zY!BDY5MmO?IjjG0;Gwz^o2x~k(=RC->a9<=dz>9R9H*&J(GNd z#!NtVyziK$SnuY3qFJ&^eEE;u!yNW_@{R%3C&XXbC zIhn&cY;1+CUcMyqn4+5fXLRs+)1eppRnfe-pBO(>EIrCuOX(S@+deiDa(RmW4gM$U zk{#q5CWbL0A$Fb|0-}bbxQCvEfM_Z@Mb7q?Mwr`DG|J@((DM!j-<4YsNX-LLQ*^{N zNSNClV#ziMY`Z$KRhfV!{yGiL1!v?bLUz|M!Gf{@TvS3-ROD&-<;{%L9&D`;2^+by zBS9N1fjw-vJs#Lsv~gFReS_783iX$qlm+W|xOCktpy{i~s350*Dd~9bnr?~GSzk*YL zeg#ds@N*=3lvn^aHSF8~R+WJE}(g?^Ayzj|as?b4^bvKd${xGpyAP+zln;GIo6twLezT(A=%A0| z7x*0v1hZOi!qrHd@|ye_smCR6vH!%gl5_tC?s9L@uKafJH9@xBnek!J6pO%^-@$@y zDeZ4xWe=Kt=N=jBChX5*#9-xiDpNYQFr(htiPn1XRbk{i%@`*aG~H>O-py}{@M5&J zaGllCl7k3%Ma5I4kAxajmKhAP3LAEp%N(yc(r|Uf2F!cSMA(`ySJyK!R*@d8@hKla z835XUGMv-}dQ5V>qebBbA(i^+6GT0detgej|u+ z_CA;)!NQbJfe>IT9~bWZN%pg@0P%ARJ z(2fZsHMS~2O|$IkRbww~bEx_@x=CZ9RX|-mY0yGBY=%Y`C_DO7O*iv6NO1QRMBy7m zd7!j5SI4h4#xwVqMxDL!_+r`HAW6w4ygtMK$p7_k`M3n%3{ItKL&A1A}?Etsuu}59o*_WM+ z>tGe9@1HW7}ZVGP)8cgQqJZj7AT4#;iJk{$R*e*0luuH0b z(7cgmK}^ls%qVI(F<@{N!B)={bmA^c__tfDr$Q7OKZ^AbA=^j^^0o8J)i-%)GBonm z*A6c}dixq-=X>UYySz{TJwA-CJjQP5KHOTDL#Fq!=L+Liy0nm8k9nYNQ3AF0Qi7Nt*p+`;Ae5%V`&Be(_M~?g zgG@WNWIa2xo!S%L<~S}WK`7dgXHVQQAi8?5ZDojUwz<;j3 z*@I4A*w_^IwZWWvSQiKhkz`r~p8@$)iBM(To?)2sVQK+Tp4mu7>m5}#xHt}74=sX+ zOIt*>$uF?pQE~K9FP0v(R9cuG)!vt^Za=YIY{qlbODMWr)PeqKJ0=FIR}dgOD=l$<&ZiY|lb$?xu%BLFT+Hkl@0F<3Ndlzw z2Sh1#Duo`pPl-+k9<8aS51bc;T(Sp*fW-Q9;%Fj}9aKR8DWi$lykp!OX$@MLsU0i4 z1>Kc}x{$HLgH>+$lE!?O-NTz*Cbo9Rgre25E<2Y8-FJ>>!*`v}KuEwJ2r(1&ZK&Dh zC;ML8+(|kMrA7vk0BB1D)y8WeJ>HbTKwwv^0X(;Sg@-_9|&Xr~L&;UN8?zf+(Qwv|V)U?09yO{fxteZvJ(sK6AMlmGJAYIh!+EGxB zLbdD}kKlUKPGV_QmnJMyg~MV)LF;p+Wo>DDhKd3c1xr~S8Y%{s>u{zZdx3_kY9^1+ zG_5|SgcuZY#5o1^oS06b*Uhzs_8mPpAaXrw!w)(Rc%~3Ff7GQ<&QVtZK$y5B}hppezYnZfnaZ=1&!E z5iyrN7>#x@?1>(;c0MT>Gn3<5&*G6F^aqUq9|2iAdsZ^?cV~&6Cuw!ilPIifq5E%T zyaQffq3nQOs3nUU;iwNkK$Yd6dbck8cPE)R(EAs7n&xc#rQUZ=_vc5!19*r)de{0J zmFSY9?x$uAdldt5T$(u5Ezg1Kc^v{|qNgh3kwW>q$c0uuUC2BxDpti;Wqf)%h5J!w zM7RIf^mkOTVn@qpgtPg5B;Tbk>Z=CGBrps00t0{na6gJP67*CoYS$dwLIU#~6u(}Q zmCO~z=p)lnqUH3D1Q%ew%UXn_^)GE@#%1(NQ3Yf6O?_v77l_rWW8l6+P2@d(abj&o%yS1*uW#WH#%&M!G8~(m9R+8}&0hq{dXl(jv+uYGb#v z^)*u^DHML&tLh$#j2nq@Z8&|I%N0ZRFGw?z^Y6{t4PB20S^~?16>3)=6rlR!Qfce^ zUKuaEpwE_xBYdc6{N)AL1SKD~o6du!qU7rU01Cm6%$r{`hesh|s2ppGclN)Ublo-u zmsX-ljXOh$`Upp;$qqD*FM$JstHHH2kz&+5!kyW1k~tOO&(-W_cHRUNSYcs#Xgl%Xh)P-aw=<$@<7>6`lIW5PjNPy|_@SX^RMApRt=W4*aSVki; z&4}>E$((OrCf`%+j(46mZ=fAB6TEbC%yG;9zBl(dv*_4Kx~a}N;J8vU5!a-6cbVGq zlf{xZnae?27)u)IfjKVdXk|&U9xgdwMTjnX(3Z)veX^)nYNmGU8lu-LsGMy&uJ;Z5 ziiS!zKtd=G_c0&(u~UML&1Wxs?eV*$S3t-erj^uRdjry_dW6d6*$NN{*xC@0Aw6jg z4lWP+Y`l+x$1zyU)n&{aDC_rMa(&A{i?SV}XX62C2Cs5n{(z*UWOlGI7rgU7aRSLX%N zzaXdqKKy%eJY_-ESP9UHPpM;*SwV|s!c}-+@zAr4t8$mk^YXhc%6>=h*Djx!;x~cM zz7vZ~&NLv7h|{_CGrUf9TZ|@@()2yFs>-xYt&4S9eu`Z6DKiv$9k2Jv`RcJO(hF>^ zA)U>PCM2k^dg&f<;k@&?h+)5je>1~S^m;7jHzx19TGP%ObNTt-ZrOP|Su+fq-C+==k}=ETKhLrV88KXSY86te@G1(+pg*$#hpED zJW;Zm@i3SjCd0?t-$4QeD~3j^Tc*q}zHyBRabbt;B?K)?dQbE3@{hPhFSzf1J5L?) z>_$Sz-l{JzhQvO9y&$Fc$t+s}ux%0)@}CVv4?^TSW?RW}puaef2!b)8sr9d5ZPzj6 z&-WL+VI?LvBCd2xZCd?VS}lK%u;RIx=C4-3y1F}b@iFC6PPMM9ZcGA1wV4-XCZS`Z z%PJItAbh)qbJbSP+Xcym3nkQEr(^3eTmtWqk%BoS+RPLQysp>z1Bw-nM&K3_TV^@K zsf9F&{jCnl9rjhTuV2r;OJ1l2Ir5nXR!U|ePK)M*y12aOYpd|#f=gFQ9*C1{n-SM; zu2&gm8Vm|4W>K_{_c82cI*z6dV`lmovn^{DoYRGonA_?Bwwsy+wMB6=Yky2a6fX%$ ze~Aztx-T3!M1N`rECJ-J5%SV$f@R8c`CZAngsgKDDk|wO)*ky0HkBOJPIJx;uQVf& zP9KHy#T{Yl&|_09BTg5%o_A(t6DD@v~xNCq{UIVG^Iq>TUir$hS##HF}3HH+ZU_(Y_44#eQ^Cg<1G=(|a8w z(ph1`&kx9)@)nv=RZX4HHyJZ-aHQzI zv4D)3>?37pLHwZF+NwhEKpg@#H4r#lixxbPWvqw?kIU?~uLPvnc>M+xZA106eeTYj zFBV*ioEVioW+&)=u$BdkuR|_fn?oc7SqC`rqS#s`0L%lEWT^yyF0SNXfUT~_4ks++ zr@0kdQy5HD5a8Qp+y}pqK>C-ZU-FXfOr8!{8FlsX93`k07uNUl@a$J@++975>nfis zb5Bj*c09ExsVhV}FmgKw%=^>~`X>}im%)PD^)QF*7U@d6{qIxMtIb@3B6I z8(pub1w`+Bz_{8Y*f<0YL*|L3+)T(g$ewGctsbZrD#)QeIUIYpJRJSDvrE`2*!Ffh zDDrE|g)5P(EVv~wPYLvTK9j|%t0AXNTJO^jF*7mD8Y<}>q*eWgrnHiPe8&RxgT?}} zsX+d9jwGcSmZ49sZBm&c*?t9}Y`k#0N{sh?GZX!@(amAsA9pcpFn&5J{EVu2S9Y*v z7#-zp%1sGSwIRBHSb%_WH6YxM#L$Us;p0HYe z;$4<$jc|;`;wCYGxZyHt0Hb`Svo!LQ8hs=Ixl|V{#tCcC;gV| zuI>faN}f+(tmRezheu9ru~fz0w`*W6LqkJNA*EkA;=UEV`q!PaWGTvlrKj(lA|Y$P zHmQ!1&;nj$F~?6C@{H8#6L0L-sK<7*OQhY{v63+D`^+n_@4nr+SQcL(H5AvltWyaG zm!4!LeYW7JE-J^*x<}q^CcKg5|NA$*^#FxDVUd$L^SS2j$Aup|%iS%SU0>9+eqK&l z+!XFl)}OjnyWF@GuO#q-`=!Z3MEy5ckB>5p=c6GzDA+qr9nr66x>onJF#@nVwgYt) zw$=06X26zixLi1B2PR4hRJ^7;!j_-kF)x+qG`y-)=UN?ncJ*rkY469!wh5wh@}8jL z_D2C$H_Aiw2{{ozn2r zk6#RYbB*|J=|z6bkO>h}C2yD>D#hc+!{1rF*;-wqBgK?AwB?YpX9{ zgXUuy5CQfEGCDbha30^K9@2)T-_k(Jg4oltsb;q$IA0|LRe*0AJ6ONDdg1C5aq`6; zOS8Mb6yUS_75-2zMWfl%jo57T!M))G$>0AA!0tfq?I-@Y+jAFnbew*NaA(OpJiV2` zvaUmJswZ-8B>0)&^}8jNQ%wg>=9qVLO9*F!#f8YPKK2VY>dX|T ziMGTy7Vl2d=zKj-(w7t}Zxb>-9onyDM2?BHs?0uk?OW)|w?E=*bL_}yi0uyBL*7Lw zQ&kpzh7BQcdU6vSMbSAJQ-wsz&-RES>~u`&h2&};gdN*5;7V-KCpYq#i&@vM5N^lH z?s#pF7ksuJnSy?>aog2Wb&JlYEC`brkw^MVBODPRKe0YqF&RbOsL>D3?}=i{i|V@G zT234HD1*OQLBZLX%vbGFm`LY3=N|N_7Z>vId(`a78~^9Kv?kAJ<7VC%52;(sReluS z-{Sgz0E|F$zltltXRtZMkHh-5Y~EtOye@wxhrgpQhCy}hGpC?83~#Veqedl073Q0r zt}sEGJb7|r+>6sGwP(tM=@mi)LI_$PX8t(@5r!M{viH9Cz4ohL{mTA)%byct?pyp) z7K9H>t2lA}fd?K)0?b718ivM%b4vd7r$6bC%B^qn>xDIzWtuF9}g6Ffj}S-2*|vOov$hz=9%ri zH?Jv`Tid}V%hlAj;{*GX7ghqWJOVCcAUvFP)>(;>ZrWV%dPKk6BCP<7 z`s&rIlerQalQCn)r0sfHH^90OG#{8Vaju823n@jcDGwSDG&-|p&63cxE}izyp@^*e z;5~xYuYdjPPR*z|!5tH;M;?B}_8-`v!oavA+z=K_*-dQIrcFus00-WcBGC>Iyfz4r zx%1{3hd5$-<_qh$Z{MCw?WkkchxAsnRrvZ`5-CM6dXT3OKzaaHcnt(f5rIQYtlFiH zrk%}p_rsg*`g_*f&z3gX$^&gSOqqZ86q!~81p+08fcO=Ww054a!i_r=T^WhH@}u|p z!TNDQyeNiv>Zk4Pf>hO^7e*_P9NKsa?q*YhQlxi(jOyAx!u> z2s9Ry3rz@aE~t4Kcs_WrHJPA2wD_Tf3-O^1fkuxWoz{)8XdR&q|D}!x!qEZ=Om{3; zkj(JV@E~|hn>Nk9{`Ie?a&gaE8Oq~Yko2O5f4b`WdKlH9dJh3)P~nfn1A*Qlz_>?; zoHemJaIno5uV}KH?p{(hH5*0M2)T3)gm*1-lbQLp^68+QY=HN z4%Oo!)pE!~7r8D>hqW`?8q_jQT?8E5R}?QX6S`5B(_qY%QWWxExalIV-SN{Nob>o+ zNPtxG>2(qJ(ew&nr2WXX?A8{%H_8laf)LVycL(>|0i9)6KjEZ)N5ku*9KP8s7|s+k z+x@YReXQ_`U9ix8z?9<0%f&rwB+znjk3hnj5$-Wd;+dKBV&SC_af#iK@7c3wr}DX= zLEbS(!knt3nSQW9yMqZDnw~82j-Qu5d!O}Js`r@FQRn0zY@E>S;Kq+KQP-@I!7QyO zS{;v*IEDg+05TM6G0JdMAV7DPLxND>!@3UdgjR3bWw$TdXunw~GpiL$S(-hyvD%(B zw$@s8-=Z-44(3e%Q>Tc=gzhW#k3*v$dt&{2SygsAZOZJO_Y*z$zl*YyPoEnR;|KMZ z;P-3(RjOaV?ah^Prn)6C>F{rm>zVwJuR^)g&Euup*V1Zx_aCqwd-vPUy$5V>^8wqg z`(69Ab?5#TJD}N8W=z|t&wR1OGgoLWQeejd+B-`$Z{+MWNB^qEII@kV7O^#XfZIULeS`poWZ8dt*zjn{wEA@D<^`PE+)#*k??^(7;dycvuDpX z*3Oh$d5g>1n`i1D44Rsnk^t)M;c#)4SIanDX)I1cKRiU?BM=x|1n8g{lhQ6$V!EIm zdk)yWk8HNzFI;ceEN-&JyIbs};njBDNDjHu&NT=fl0KY0;M%H4s$kmMVML5*{ zq=HvyJ4i7BxtheF9%vcZLQ-dGa8+Ik0;5-mvch1}EXI3}80=>Kxb7C?-F)Dn9gzD$ z4eYJ$0u(u?z8Jh=x`)+NS)CZ`5ir=bHCC(p;kvJvdoi<3SZ_nb!TW(C%81aM+HQNg zds`0Lx~;ox#m4QnY=dm0H|?-xTXx&S+xOZ-dsIpL#rz%+Q&5YM z)yPgV_%%H~cUZL{^8Hg2Sil91xwVSu^sOI-NQ>T0V~9o6d! zgWqjZUMj6h->I1MgNj4piLYO%O*mKT;Iu6dTE)5#S;e^X1vlpxh$*hr)}fW#?BKTL zR^2!``RgzbNr35{Z9c+d!bxQuHmpX`_ft7L%=nQx(md645@B+bB^HyT0bx7ZEKxeX z#VWpIJrKNWeTrFaf`;)){#+x7gLn%}0iuC75`v}TR99(-trHxY0OVqiD-t)L3 zL?93d^brEt4v!8C2I-&#ilwWX?9caauwUM{$$qzSuT2%RHCuwmv&Yvao(;jnDce~g ze2g1bWsh#zXWzQ+A@?$*uQ?Xciot4aIbiEIZn9HmjI?9t&P$#Dkq#tX0mtA%{biZ% z{hH<6ykn28-LlhGY~E?Bwuk}OZ|#n~wtUxqdtlH0JkyPuiWyM`f5v|z$`X~zU@Zp5 zs&uW^RrPw3X8Y$f*4i{N+_T0v+KlldZKjy+sbaP#jT&yX@<$tjm|f?#KQSO3A8gT7 z?3GY;-(wr>*2Qb>rez!Lx7rxNcB{;^SIW@-8>?~+uNT8Dv%#dfAtF((J@5IaL;t2% zipG(mtv4&)71E@v&|0YLmv2G(&IgiE^7M&~_MABr?A-ZN?ew|F*to{J^bTB(batKU z0_Vq6zMtN>$YxI4ZdEmPR;AhSJlbfM4LS2nA9A+ z9t+;Pb>jdzN(zMFAR(h=`zmW&^bb}$PG^8rXr)*ZK=fWhQ~mg-So1^gvgY0IwdyHn zTUFy!y^whcv)-!L${MXHODYTE?M~U`t(^$*$9q=W#+_|xVQy9RFg0u1X$QiLq6QNf z_t!jouG&b=5n$mtIPzwUBm8qqCE{EII6U3WM2qL zN8INYFrp=uKOHp!h%*KGE+F4g7V1*<$iR^&Zt@J@5jPf(GOmF@HxYmdVwB1cp_F-g&^@as3MAhIS4Siu>#p z_bmDQj@+{EviJDq8SNv{yeU1e8nNMet4&6G4hkG#Wt+mXqR0$Jq7R> zi$nLtGqdUWx|2D*v>6Ys*kpG;vfgfAzS(}Zeg}@4T5KO|VY=Ieh>>DS>LpCp43pMI z8KG~20f(~aB?py@`b0iyQG3}Zfn|}j2aBXdNX&!Q0;LX3v&%NJ)?PYgl$|wuf<0~C z6gzp=L>nzLCCKxd9{Tu?3m*sr*Zt*T``XQm?T?Q&<(aLzYMUg+t6_AVN|II^$UIQH z1gT61<A3JIJ0?2ErzWz*)j*&1o!VDz8JU}bBzxKt{k zb_vp6NVb0I8awCIS?yV%$_^QrSuafeD&Fe|{1;SkJLhlN}IdSqLCm z!vCIq`|WvWongm})PaK?EYQySALx5p7ee07z5DF8g^R4UCz`t+W)}Vm?MFkyaGNrD zysZ05PY9tusFW6K!^W*zJdRl%YkGQE&_48D_<9F14}Dmip=*Mb7dWUT4!uF3$O{#n zb>5Hnn}sS)I5S8?ybpmSK@orY7=e6S%?llA23Bot_r4ap|IsaWgG{V`v1qF;l;CiJ zW~H8?#pG(v9hz7jE?czZS~0xO8pD2AM+1pb*G#R9ZEVACnJ3l4ns(L$9626ouu|o0 z-M!C#e#cV#?w!kR!J6%PrdtemozkD!I7~8!RPag;;Q>Y)#v4X`x2_1FyyNG8({0fo zdMSReKh=tFtOmv!nO`xZRHJ9!5oeRuP5k`P&Gz$$HYmV@cKXzj_Lis4v6r89f=w8y zldt8culqGcnd00hQ>%ad!Jq8Qe|l8g+SJ;tQ3yWLhRDNi^{d+@NNYGxdDV+UC|^^D zIQ8fOOj{K9zC4h+Q(I6B;5AM-4VTFeSUoH)%sc<@ZoBfX753wITwv3sG3pCUY9##F z_+d7G)G#|%T7^nX0Xrl)AhgjI3ECs&U$-Sg zZo9u^$&$1q6uVm8dh4wze;m43OqyS?HG9pTcFYKSal>BAY0KUubOE_Mw>Q_~V7-3M z#+kkM2%qStMe=I9n|3$b+LnFx;IKy1=`d_b(C0**f?r*&PCP$G--v-1IVvGig;Z6w zS@X_4+Odotgr2FdYpMI?Eu>sxFm=BKhx(*f98#3E=kj)xY4d%zt<*6V3^l#V%4JluQL;??q+~=o826xyMfBo zANaEVAhGn)5NWLIH*)ix`ySh9FaFNWwo(T^nuLwnV!C0mo0XRX`fXEwnB`YBBw`%N zjwQvgn8TBoIxOxk_CF1H9SYO?&O`pWS1Z-n`Tny!CuL zZSIsL^c<<_k)ke;Gw{Eb=HtsZK4PcO8f`1~x7rr9JId~5JW?T}kYG1%NYm5Xr+V0v zgdel9`o5-%8N6-zR{PfV_u404b8a_cJG>Vqp&5JCxzgFEGFQr>CyaAyy9(O|L$8_t z6kke~h_fx`FnLLds=Y1$qE&7Blo|x~zWg%;9smo4JXgvIT;AI6BS*M8t@gAewHmfR{WnL(= zvqH$hy&-$m`X<^WnAfRp4v^>m%#zwCZM>~@uhQA0rl&P*`s=mqY4cospLO#rdN_af zFg*LaVmE)zzW2D(Re2LSY6iCw-IfkPtxlXlTT!4qZ6dxu@Cge|W7e zT(ZU%%I^5~Wm|0N!!m)`r5PMXVrr6AV06&>(CFk-rP7}aue7CSzXFq#nhi#CF&Qm0G zps9doW*4i&nOGf7*mbw8V9Ws0-$Z`^1J4dlum9ga+sf_x?Dz@8rG03#bz+$KM@Osd zr2KH@*Vj}WFI^d3+kWLw$H}EMCusJ6tn#*0n;1Or@|$hxM_yuMHPMn~lKLuNQ^x8v zZE1d1Nt-uH>F!oz-l+^RhYlWe!_Yv~k>gETq|bG@>THArwB4A*NxGk{H8cG|#r4vi zGzi1GmnUpG)svQk@PamC_d{0My1*)@z9SzYW;GHfs_9)SB#^X8AmL+y7)%jRnm5{x z^&4!|#IX`Ws41DU)vS{WRT$xibt0|%ta;O;!tc2@{+QDUolh@+*X_UUDd$V(4@IGI z?&MXG&jEgH&(!$w;{@UOlsINr>;UEFSx?9j_^Dpd_~c~H))S4<6XNP)2l^_sP_2~` zdQ~=xM~mn;0*drcZ7d05w6%O+l}$5S_kQo!&irNFxj&r$?EAufd;T8pt%_qEoacV6 zlL0JJ*$W`8n$|?{s8ZuSsKM>-hgR8*cPz21Zh6G+y-x=MX?Dd%tC%B(*^C)lj4AaZ zeQla`RzpvHsQ^z`{b44P^AwgfHQTa3JZe{5wM2txrJZ%=Bzx7fPOuBkKG9B`GdaZ( zg2({2)FGDCcFi!|C!zROZJ+zcrQ7VLZTsywwX=D{t8Bh(p*z~eYKTEF@-IeY$e`}y0?wU=Ifw{6}k zyOV|*n}O+*n0|0NprkQFV?I6!OS-~-kXNYqns@E|FD3EPa7B104m2N>9n&;SU}_Vy zt&^Iy@OA56r7q#<3$HlME;;XH`GP#<{8Rb5Rk91u(#nS$MY2DyXV zt!>|C+p~SYExT`(O_{|GRf4I0orDmLnHqBsH1DZkCTN zsXtz0eGzBG@i+qX%VkLnoEpcH@Uc$=$sG%q+4rvftNq|N%T*pVzDad9_qdVT{Y6_0 zsxj`>4A&}lTG+$PjoxF=!j+bVCZv5q=twbvqjf)5rW37Nz?nNngS0xA+aKFzx8HV? z*{ECXUtj$s``Z^j)t)r(m?Vg}5LA{xMy3Y^fnAyvUbJ$%{XxRVpSCvJT&)k7B-K17 zGc?V1K6X&@7K!!@0Uz+j46m~*UVFBk^}So{VJ&pV1Z$Qw0CX~#{jf8WtX>L(S$_>` z>eUj#W}>sLSDUI&BdK)h}&$03Swhly0jMDs@ryXyP&6#4qy6X}9 z>AkD$&#Sd2hJF}aX!Gi8$7lj6WlwptN0FM>e7R{eNNn{9rT<_VJ;?bT<^wHKW^*G|>qe5gM(K`G+F_~W29uUG0q zC#MXDWQ^92j7sBSbwkd^PpGzOPj0rh z#c!~K6aQ${V~@+Hf!5_{A%|1YbuDxpEr1MuIa{QC@?Gbs6LwW&H^iP2V#h&+%kfkF z2K7AxWhsE<_l?eXaFO==}PF9J@W?6OvRy(rBCTp;qF8>vy z>a0bJIzRpWh4$(1-E05$cW2n!|MonaGESxj8s+GVq7&--MmiK@oIat^{{F=$*_$ss z!5+|J^FOlK{J|}D%a;8%PlNpwEw+>?ZZf~3`9}X&;;o>a*?Y!uQ*F)rUuZv*7WVsh zudsX8axkPSP|Wry^*56=*@4A3eT6gFXabldVTjCX11K>;vz71dz?p6=`eD9V*j}rb z%%xyO&3oeCsY#ZN5_VI6rgtY#8EJ2nCifL*&$k)$TaYuV`KKfP>rDKEt2Y}$*{lg8 z?OiW8-ClqGDYj(I7Q0vLWbR(E*%qwbVfSxSU%O8WalM>O1o6wyF4-96tpO)pQ`ys> z_ADw4!B}h$hFh6XVxbR$2KSiNBFRA7d25EY6@BLHadyVc@pkH*Np{kV2{usz7_>_} zmMvY|WT#8JJ){T?N9#5LMT-y~39$}Jm|z00y|T#KpRlghxg7IOZ?usU$0SWz^+*Y6 z4Z++@GGW10Qnfep5DnbwYsk@>(F zH9&32Hg(1bEtYGufB*WO_V2%2V!wX>bM3s4wbR6qb`fv98h8ZY-T2};L~y50?w+cF z@6wCTu-D3@=YBEUe_f_+cpuxWO$Bz^4*Dk;P@w&5K zg!E_!Ea*OY4&8BFrq$zv|;-mYmiVh zQrq%+8Pba*{6pSg%x+)2#@_zh2knP1d5WEL(hQrf%{`tftua4no7U-V&}Ja(n|9l3 z9Za}!+iqKbh31`|w^wVBAw zF%33Bef~@dT9dUNWs2%${K(-ch)2<+9n0=qy4F7N+xzU~Nh9rljx8YpBwJR`aJOr! zy+$#)YHQz(DOF5+L{EqtY!N8bKA9~}V_22Ws?cKO#$7r~r!}o%C|}LZ1ngU zwrTA>$&B!jw$@43saGa7T6gcUNqbtg3s|L%-zTN9K94x z;IfSUumAj@z3|h&HQTh;PM9y1uxjHW?HI%~*g2<9vbVhJ6g%VOnKpasxHP~%Ar;jk zcC6}h6EVA1qk*_yztK9-%Z?dqr_G;XuYCR)_DKmk_by&-KfL~ayXvMDs`;GFoK&v` zl@&H)-Y8q&++r8K;wSdekFBzQ{k!LZia=g9oM$9`)^L&-ri-32%bx%A znf7j(s@x*s_quyG+Ar7bv58_*=W1bky{5P^(L!_ATY@h=S_qt(`|XmWKSD6_^;cm{ z+FkEiU;DlN+j+;?bDuKDCXa5kXPq)L{SHV#fZ1NdL4-Q_d$q31b#BK7G20L5Y>>xv zXxoLdr+)dRYWBKT#Zh;RK2)L$G1Ab^(8=CUlF8^49Yi==4EKx)Bho>H)5niW z%r^%SI&IP<%Z>M~uz$Q^k^T9d7vu#H^@+V<9zD*Eu8lgO-L%2NB|rX)%+c<(_s*Ye z&phE6J5id4smd4H5q|ThPdFqvrESp`zPr@7?$zn=Xq{X2OY2oM`_6k{00-6_a7Y*v zO7)ttAE6z;hSgLHn&bmthtNJ@yk7P2@Y=0*=fmsmCyUnE5+YW8xLh4Yk#|!hm{p~fND>+VFHzw}Pxujcg>b=ChcHFR zsB5gTWp{0|^~;;Awyr|wsqC;}!naawsS>jmong_|y4@x`=LTz-aYjC#9-3d#<}|;O znsdHBQ|6xVhx)7%AZYr60)b)#us_jPZ(|RtZEAdMQ~%h&`Y!dYJ7rPO(mFyK;+>kf z>AL;!7R_-IK6bZiCoqk-RoZ<`AMbe0_UzS0SlKV9I3NA}JeX3k0C9$XzXqYt zT>b~U>`Ql8!`wzY_V_XOzy=8(l2YIHk`wH87oKLPoiI&%yknS@v>h`hj!wT9oO7~$KnFryp>-zz{eKTi9Imn9TG-COH|wl1 z_K7bquzR)N{^cKgg-z6~o98(jciaa8#RxE-I^*YBkt(&*<7bRXzrWR~zbsb2;ofz2 z?V={TT_#$b2Qo=By@W-3=S!K2Ago^~e^%uzuamb6V~JyQ)bsbtH~{-E|=B{9Ow zb)xwjuDsL6T))WPb;>bzu6EHoUJN!nLd}>kD*evV;%q1SryV+&tVK5WyQDqYrkVeJ zn)QbPXJIQ7C4~_$wG}bkVgPHzWTy!eX)i`f_^1){jV2?VKjD$LUzyEYyLG26mHE>x z53RNTd0>sL+oV}>EefwuAJ<>RqI@lCvQx)3*rU>BeCe)L_NCibDBWs%x=awCGG(-# zphFU;Nnn~Pd`9U+@-g}i*KUKD(TpH5W{VQBrIoI-X7#yiWk&0tOJ{NPxBC;njQ&osf8#HC0mK>kEn1^->F2Jo zAN_QxoqXCj+a!kKfyLYHl^4#ncfax(cJ`BJ7bci6r-G?tX3P(8QrBvGG`KUsZ_^AQ zDrKZ*i6S;k1Kxzu!_zjq4RxB~Nu6LG_=9PmZ)r#r*BN3Q_`UMEPqxdxbCcbC+gh77 zyD{x?Kj);;cHPY@?4ljtw`)K5T02JO2A=o+t-=9_oxxB)Fa@(tiC3L{Azi)m2#fs` zKfAx4UOgVaX2MhdschLw&%7VVTk!SqJ24!>vQaaRXP+`9{ob}^k1f!N=)cu^ksmDI znRXI7Ls}6wPRLu{bXhxTLLCjtJS{*zDoAaX#fOz`THJr2Uf*~Bba&pYXh&nzPmC5r zyG~5-C;q(5?50Pw3tyeRZ0aaGb?R7~C#HLv`i6<>Z^mf*TbS)}x@xJoPHaCiYE~oI zv1_005aYdZyUdxk?6PGWw%dJcw%Ju{wfI}w2$=3lO`Oad-(Zi3>Fc`199e!+Xo0g+ zOlFJP@^oov*okVN7|8`Xk7R*N(vmulJ_8Mn+WKt$rpknBf&_!{Vkk!IS0~MMtxT-= zAuzGa7Fff)GAXrRT8(D)=R2h_*&+NYI&Ep1KZuaUu-zrf*Vd|YBw8zN|L!_zbzD&N6uJ?U z;z%6*uvy~tuTdG*{(PC`abhs?DeXtwHFwLL%W8o@((~I^Y{skeVQs>xAQ+waDt;u$b87^xXHLnh^ zN$pGFUGw3>M>XHVhz8p@e24AYrBmBwAWOe7SXB5l1~U_VSor=Kk^3lv&cLsixx!2S z2pq)Y{$!H`n2PYu@h!aOuJ|i7Yre@ zS_m}fop!AKN@s(7?f?F4AN#e3m$uTe(%kcabY-t~!L{JqEJ{yK}(-{-&C*4Bfz zPqx9EHf<7)Ohc$7gjyUCeml)d75@;uD;|dmqZ&HO^t%L5aUc4#K=(aQ4 z^BQYyo;J3as_STG|7HF;R$7b(G012*FePH*1C1C21IC01+k_vGfU{Q&_cpX9Vz&9M z*VfLD$fRqju8HwRDkZGqtJUn?GzlGadwUemR^Ao0k(5Y3{2}-1b!{^<*s6DS=Hfh( zTFR;XVQR!8e0B+YOhUtBv!k z*|D|j`gQAV?6@)MMbDaUd%TUOT5k0Nn!TMU#m5dQIof2>cF=Zj-KP`U_e!RylbkY4 z--R?a(tOBKc~y?|XO9>;%#&3adY2a!!u0?B}q*bdv@r-{CKs|I!zkp16mCNh0S-*hJq%fRYF9A#y-8V zoicMS7D8x;9ce!D!biPrlEuM3d8$_P=8Gx?DwTX3DHbIwfE0ueElzyl`>(Wn9&WNz zWpcG}$#(mj=N)UG{rl(HylI+|5n&^Y;K6U7X1f+FT4lE^e9XRl`*Leq-@X8^v08(B zjg7^QREx>?se|9m;HHMvs5sVR8@#pEZolDS>F_mNigS4JlP1~UoPAspIA-aPDt50S zU$nyE(v$1yZnH>C+PCZwCWsr5ICYf~zewfmewVMyW zOD@=N_sT@;`g_*dFBUi15_PmE%fw~8w)BPEM2mZBj@d(IQ|WYvKC!ukCJqi(r82j| zefKkgD?f(24N4Ctw^97kF`C)0(WD7WH>k(mdiRLTgC5x~6H@X;U*tdX=*()WsPqR{ zRbqP?@C)J3e?+eS2F81Ywh^bUiJG%EIEj6=gbreA&>yY?O?kF4FN~0GvIlzLE4%ZevH|d5e{`ew@*W>a0^hckD(8zb@ zzZ$-|o^Z^(E93Ap}i`NfR;xO^A3lUlURxA!LUO zA?g(PL^=~bs%8GUQM*y@S4UAQoDcmNG*rHjk`zFg{b9|j$9_So4Y?1w*Cto0~2**{81 zDOd8D_gzbt*wWKax9U?*u-r~ zTFLaFz31T@q4ONA+4J*Hotl2{+_=l`ShT@@wQz%7xpG%3=+mSX87}PzY#a=i`sVZ_ zEcaZ83eh=xak$S3BXEM@r!7<7fg}bke*2X!v+n#uAR!xQ0mh2ahB2iRrf#X9$C={1 zKmK~Q^JROSA(!_;e3TKZei(bUq~!#5!jisYLOOo0+d(5QPjPwCP#`Q63Y>EIc=>ts zu%3A@k6+zAdo3R1isQ)o?W=p*T55Z0fBCjYs>)^le=3LPx&2K_I^|7&Jj~LRc z3SdB|ts8dd`smIKcV9{th7kt4g9{sk}kJZpwcU)e` zkl;RQ@=O~uy+#KRN~EqBrZHFMo(DAku|}z?+8XCwpouB9X0(gF7#9Rj0VmpUg&aE8 z6!oV35$0$`;Bg_OT@ymbs~L@YnPj>qB&YEoJ`S_1$t*Z;U-Q9Y_9T30B5#jAp<#TF zr2?d9#5<2t$yYD)QQSB8GMc^%M#oDB#Q6Eo|9P$b@%FX$loQ9=!Uwk5*FO3j`-fLP zGbQF)4<7D$c&&Z>%G>PPzpPDXcN`~EC{+84wB==!&H|Y&vy;otn`h6|g3x)|fu%uq z*9a}lwvb+nv>z=JW;SfzX@Akyk=HIECa%GqelNxu?xd4IQDX zEU7);6{9pd)ksS)RJc|?;_4c!7`Ge_u&k=a2Wd6m)P{MUF-Z%Z^?S)VbL^j1G}%ou z(Yp5j&35CK=Cp8~8RlxHmD_2kRAZq$exoa;q_6JLtaB!Of83HM1OkZhuvr;o3tjT* zL!8%6@ilo#zOueORQ@etJY30j>|TIgT)j*L&EEGSssRl`+sl4qhumj!k}XrZ*B`!X z=!n}di7K*+nnoM@?2lW+Y44IIZlqOf;w4KBFZ`-C-G8xdd+g^MW2)vH&pVnO*85kan~fb>A< zB%~1n>DiRs>}IpQ_mcKM&zx^|zbxt7zTJGYd%n}=%$a%5%scNpZ_A>2PE;HMV@8r1 z;{x14Y6!Fhp(^J?#w;v_e~fepPA#c=U2fy4z1sWK0Ea*a6Cz0n4b#Q#a{iNdEFVfj zs;K@d?T*gOhhKOQO2~XPkzAAS1#jv(2zcx~_5Ysz0FOVl8E4Fh!@3RCcN-JilgoD!zQ~I7}Wsgod7nBizG4 zYIIzQXd^-}kt$U;oj(QH`Q>=_y-oP}TU5znr=37XWFuX=UH;G;_}1n5c<{Efjmutn z&62m8Q5gyN^q1FR%AL=c6`ckmR-^{vJHLAi@BBWs$6Q?=){Y)KkvagAp>{<&nX?{y zTkjRjcBoC9Py1_fpTM<(C~8)!43mb(neFCtTd;0>8Q$4Yf?upF$IJ$fje+v{`s+t? ztG7PIu|CJq=0E$4-Mba(Jv?a$I8Yq0u(V7F_d${I4s?hswD@MzVE+KbO}!Y=lNO_X z??b5Fn}dDS(ei1oMoVxtkKaXe-A5)nZR62G&1V0RkCFKp>J)Jb(hvJ?@7@c6o`Qg7 zLfQ?}Nv~l zM`U^5qtr@1c+HdX!9W>p9-|Ya)mt$imGoTo zrC%&T?AS1@FKED>Ul@n~xOt)J&2qaa4;Y^@vT(+I|HM06%aD~Gj(UC#=|1OIwUF_c zjHN%g5@%0xm=75_s%-w7qz`BGt%-oOw@A4CzH1_q?yV5=l&mED<3BIQBVU?{oN|io zXd^NrH5{Mj*I@kjpU3us3R7aYxhCnHB!-gIjYy5cg)`GdYngL-g>vHF^zHuey7#}2VJ8DZ93h6f zGvN1pl8kljNp$4z%Sm4=jkI+MAV@roF;h>|7QqE41tRQFYwUl0kj*%uM6izx&ieuLVMp3P!B8F}0kMGn!#|c`$L31JWrQIkGZH_#bFrDo ztJNHaGaAPA9@l zNwGp4M#P+&A2A<7)S@LMl?kRw%eStWiA?(NzwD8B>0#jzvgpz>hko@(KlpDH{QNpb z_6;|F>7`?T=e6_j@~RxHAcHhAE)d&4slsa?Y{czX^=~Oi2ZX#wo&W)N!rd|w_FCuq zIoF-5meA^)r!UT(amK{}54h(qZE0cE>xoB1c&inC zJz-c#(hUbAjtoKsY3?fNLW>*i=43zH!igC;Z^AAt-CBY$#-2$g9@RpT+yKi1mSCcg zvF@v2-3=$(I$?WhT52E0Erjis%jzc116vqhN2{aP&mGM>PQC1N=E7<9qj`_5?+gGJ zBhmp1<>{QMITcY->Xc)%b;r+;4eO8%$jW*pTxG z21EqECc@3Titxifuf^Dr5scf>i02=^+yv~B&wKgA-)sgtXuy^<91eNXfPgsbv>>=e<~tE7;uTmyuT4w1|Z%akAQ$>GQ@Q3r$7G# zbfBPQ3HT%Rts`CYh{muIwjFAh;s7`?6CASClk08aXX}_46^aL@W#GrJZ$Vr_2r{|y zFL&QEG;byv&{s06ZQR>=gqOQDxtFW5^-f{40Nx1VBpl=-x^s#e@ts8@O;AKtz_vlh zT}>)W-!hJIke>bOxp?}6&G_+(UC1qKAjd%nDBvKO5mto^i6gzHw`Ai1OX$)v3aY=7 zZIj(QS37={&poeTtTt;?;6ue?kt(i^RfgiS6v|GJ!EC{xjQ5^73}0R}5m^b5rru`3 ziupVt3636-2)m9i;Sn|RR>BK`6M}$xuC!nd);A)lkuLYCf2D;8sb-0ZmO7R{7h@vb zUG$z%$=*y|g@8veAa)h*56><)Gv$`zW_2(e3aHS;)0*Ore1>~jE-Bu zz7uS4^B}Cb_X5Lw7_U^EVyw^f-DEc2-I#|bKhDL!Fuf+Fm zm~Y;#Nn_^t41DdXG5GzzHeu}0Fns)B3D#`bjdQ7@rTg7E;X4iS{+G5C@p68a|;)GoEDwY8PO*P7t(&j504)3KN;Pim>^FT(ZyD`z4#Iuv)j zx7Bg$%>aOrB)F4=Z-{ynY)>wE<(gNMph01s8lnmFMADI-)pL42kUs&ePayG@&TEL{ ziWC{Vh2(uy5FWjLDsEjkf&MidW}&=P%@+jiaZ(cYR*UTP;)!BRT!PdQaHKzi* zD=Bpl-X)W!K#A0?MFOSAs;}T^urn(z7->PtftA#Qc)^vJjcs$L`(4z(yvYjzF9c2w0vgR? zKHBJNSImcra84CNKQ43);$m_egVb8@SaQ+J2(BG*LQd{gy_0)@p@BYUK&Ml&UaIdNlApiVE)6$-$6d@E;5tCu$vP_Iw!4s#`SpXf%7q)QOhM>6S7u* zNj#q6s@qrJ-X_@^t&RMV5f#V?m=wdYN2)xMYjSR7l90P>kww_Owzmea{dFZqC2hj* zuAGK*CS;gA>ck?3l=o6H5kI}Y694(PPw4hE7#SJixcAAmm`}Tt8DrDh10&vk`3(I2 z-&;BF@r_|VSpLaQ!+_XH#d*8;*9(CYgMj*{Hc35NT3QT|t|-d`A!$sA;m{_=N=%3( zA!0-%6;bl!aih8kHw1C9dTX+k{XLB)q*RB|Ln zr^L{iNDPu=q=@AtF-mWJnnMRBwfOeM)9huB*sEV@Q&jYUmh3``ZR0>|pSL}I3jv`H z;lur8KH|xIw8!}%9XFU>QOl@ug`mL*cLggHLY0f&L5G@9LzPS?Vmfh)@QpY12n70^ z0kPlq`M1`yGx#8yR&^%?%OWCw;1jlfQd&ok-KU^flI|M-Vndvoc^LipT zMJ-Pzvl7TjE1QhVqTjuTpIk5&-??-;{8_dpR@)W7apf$$zqJT&WtSr@l$y+8et76_ z%kk2WuVXoU3-eAKnS$Fc7>>Uz-2uPkAUwAs7x&!KglMWJh*(jby~zuK6M=yGizaLl zBF)Xs5aSW(fsxx!q%FtGBc?;ler;{787GqP2oHGbc(G%-m{QueKFh@=(??)PbQrE< zydZ54m)9C+6`S^yW5eERtS_s_##*X?GQN;ml@~yWf6PQD;@J?;yr#X9MH4QX+ab_- z-7NLY7^_WOX~{Gfq)jY()3s zBh{02Zfa@6R~}n|zka+MS5HdCoY5&5ksL#{*I=_vqt`$s5ean+;#ywUh(BPeMlXy(Wg{pVSB$5KkN_kS$A(k- z5<{Q!+D&y}2dTSi6(4a@ObLrH>2njdMq~!^b^uUme^2uD)-jo%UK^ z)u}t*tA!M88tY1G+dyhxcf!hB*2zIYDAPtbk0bLDdXND$xk_QYT@fcJ!g&)j4U%^( z(usx%A#OQDJRC@z6G1ioTB_!2(OQcfLTTRA{SfF=2E?wE?JlasPhZ`FG)6Ap!-$mE zas^RIh_{fF_YF_1MC6bV?BN8I%+71nUIa%+~3Y#YAumGG*_oOMDuw;a|v9=l8YiLo@L=#QgZ=`cVi{gY9 zm*9)w>-lvpf@ytd%{+c@zCV+$j>=MvQWc@xyG~IcpdLpHjVt3n^)F5oTL7CCk*f zhIpo`wh>}jDu`Pu>Y7kYqPv#Y>pA&VlQF5M4*%4yix6r&i*m97-HkP!;OQ)j3#|TZP=8rI{?Y4et57 zdMpM3i6O`(e!b_FjldgQFl$Hz&di9#_>^c2r3-UOB4Slnu1IR@iPS(34g^%C4RXbU>WMm>Y?+oA*Itsqs;Vlp_@Lwdo-D)O(9mGo zq_OU0Jo@`r^_=<(7hC2dp7dm}jc{(_d<2vEU<^5;CM`y*FE)|ZlM{+?N~wKFGios? zs7aP+ji`y@Vll^{%JhijF%{_^X!I!qVyW;I>vmBr(TqLufw=9gEDTSNHyuMr>4{ev zWRTvbB2vOpLk?By30%&?gV)bB6SDP=^)_ui&i%vth@+b|KTgiOIN8R~*29-&OoFCv&k91-6EuBJ6GdU**6f=_cx@ynGvvA(Dl$z)P;X&E_cNH`u{ zmVKDdi|7;k~=82AlU(U`0+DUd*fFLJRRBrMx1# zRc4PSN~|M@uLC&2>)WlO^k=FESXzInk$Zi;D}1+r-dF%B?C3B!Z2`MjIK7Mlx*;tyJA3E^laVLm5@S%E{an zam!&8~>c zKUvrn6&0B=ZKa!DCgfxx=*{050vg(e`Jgm4u}KE!?vjsA%!lh?6DLGWh$JDjP3y1; z*-xC+ghon|WvH&Q`#RY=cp$1eCD8OK10p1<)x9OlX{$lz1GW2c6$AenAaiEiRal7! zU*C!t+F9(S^dp^}B)_~Fzxv`dqszOWYE+UkY^K`PxsNYH6!n}BQNmHguOmr`F+k*0 z^~jPqv&Z63@9r`NkExV`{F?y_Kij?+(?+M7i6W7z828Ld$0ILpVF&la zIvQbKd%^TxUGdSxujEDW1`Pe`N8e7HRIg>aS}9`Fq;7iP;hZBI9(=CsY`V zh$?LNmpOVtHdkWr5b#O=$}SHd~9qe(rBoj786Pppm6WCwR5s8)t^j~Q zf4>o1D;ltd`pu-y9d#|L8zgWGCDGkP2Ef>f^z>+8)%UZn7O>7*yN^}-$WUnYQP(TT zJS;D%#qxq`QVv{wR3nLBjry6S)iU0fdYaw4m=MD(h>6iN%CEi734e1ekO}5`mfu5? z?;&&s;7`VjIncs)*Ydea=1)-B9+)%+G)Ic@4xuV>PEHQu;^GWL(qG5F7)G^8%v5=K zIZ{(o&A8Szwbzb&5Bwb7DtR~rgeSG2ESnE8AJ*o>QM)=a^I@CR!Gwey#)Je@BJE52 zv_ZrbvI+5~?u9^~F(B6XJdf&E&r)?MHOvo^gA<>}={?J7u22Pc*xoV*J;F~>&)Vac_#a0VqRN}9Cz?@F@g zaC(%(PB_O=K>_$CRltTPMBu__Rw02aWo7Jq!CWo8>iJJlFmec2>O#%xn+iIE5{@%R z$6{GtHL_$2!gwSrH|4t+5E-rmbEahCk*78Q$pP3v4=^okM<6FfRi&rAkiBX>~F?- zBZ=g-Twz;VLXUj9FT&Xn$wU$&wz{HO(DIvQDrMluS*NII9- zt57l}e$=p&n_T5H=!*FaKf!v6QIQF@q(UuhJ6lXAA_Nx;@n>c~w%3>Kl(bt-mwkvp zt1obXEpKy`QO|N@L!zcDsSeHrXs?*d9`mCyE~Z;5N@B#TtE;hY-8#&gHOmm{mYDaL z3Lm2)7ME%K*45SF(@#G|OiYZyXKJ6?=}le;bOizj4l&?vLlY9IcGbar48~w%^Wmm; zb(EJJ&V)GXSclkOWtbkzGV=_^I?BV|%clf^K4UPT$LQz~ zym9LSq*DT+iQ5uv)85S~nccU!~V z3KG**Ute#eKq6E`#CUqr7*mXkI^uz9lDw+1T;s+QKS@PY#IkLLaE^#=x#z9qg31=+ z9YaC$n!yq7jWvUp7q}lp;yspIAjh>Oyp>giwp$EQa){hvwen#haKzO z6O-mE%YcXh*BFS6jYUC00hTRWh8Z(v9EJ}(u?@CPwk_IMR#sx=%9X~xOEzA@X_D}0 zOzXZkc_DC05MbXqK-JQ?`uzwd*4RfZ;5X>RnGgHU4ko0-CgcF=xNw?;q_iAB2^WPd zj@>24>PlP&qn$?^V{D`vyDEI0bxRRas5XFz0Q_ua4!$sd9AarO=`cZq zFnweapQ5Wo&Z5y|LjJL_6#vcgGhI5~LuiDGq#!7uVsQ-(O=)ZzPXlzP;^&P zNpkV2@_+kg4jz9u7Y|=I9^)D3M9C`+F<|icA2Ay?K8DmVPN!rE5-}atV~!ZI{=$IS z@okZ;i0ysCF5C;phyYg}B@xNW7fu{T7Rw=;Ie6XigYqfAp0}&}iYHs}XIzs{8sLm}p@BOP|qaoKg`-d6XU$9~E^r{^52|BEmefiACi zvI9v-NPui7-gx5;OqehMX=!OjLgG2KK5|WDaK3ZrPNPO9HBw1GB-IgPWsT77ct3Ip z1M8~#qsFmE4M&I0d)wmkDNCevRh0!qGPm*Ph#RJ@A3!-}VRce3r ziGZiCDtQv`wR7W1Jks^78hBd0lPc1C>kBC5p!~hG6&XVu;#hCk!Kmj(O@SXhj+1gB zJy=~aArWIT6WR%ocNy4E2I^7tcvZtfUsvte~WZIuK}RqA{eBdXrm6l7VG+ry$lj zJt_dt@2bIhzgvs%&riV@77nAA9NOG4c^Q%JhIsdtCqmmYAR@qzIYGPfcHi53zobD& zk!*+o5fdgm7z>l$uU+@NQA1uyBVDWe?nx7*m=qxf*UtU+VaM*6VjvOShc)svKd}sm z2y{sb!otFcuN^X~qM`z8)~vyX4I2<09c}Je38^Q5O?6jY*kUi2;%H zLrj&VJGy>M+~x@l>ef2Y%OB|yxsX0YBWV&t9P%#A;6AW*$^HCjb?BV0^SPdL75};N z>~-bgyAF}UR^5ChQgxL>`n~6pd52tTONuM$g0R^sCL+{&+Sf(W>t2ALKgxp#qm|8| zUP^oOPp0G+(uA6m;wbs34Q)gL&FLIpj3-lyH}A0^&?gLt?b$U9NcBNs9fnX>xsW=% ziS$RWq(1pcc9BtAk_tf}5fl}lF=Ke5iN~SADqZO3KFMP^bU&k49M3_MPxXb1hsEK1 z#s*RHabrHV7FS`x3oB7h>B9tetc~JoxecW&-bgVGWJu0`@ngKr4tEX>$ZZ-kBo>QO zBk^`&4TjN3RB7$@BEx_X$C?yOyOA^ELa?r|7QP|O!*Yu%>die7O#c3K?MqJzE4GzW z-++V^{n|UpHNtr!-Q?9k5$7us_KzGHap`$oy65{3j<4l2`&S*z3O)ZVJzA< zbYMA87*R*BgD3VU)yYahf`S|gy*(F0B6|ukBZY;9MoMTIbnW{hSs1$4TT+hr_;|x~ zNQx`Thr9Gc?Xl1GMV))|orZm6d#g$NlU-#tT@B?&2QCHu<1(`;+L^>(1_PAcIGaYtLN4hRU@p{rSKKq-|+CUnJnQT70 z?_5sjbDK|lQE_!FulNub0DZ*D^lh?m=%eA??Tzmf21LLU@m^0`h~`R0bdKbQRYXdP z%#>Pb!?BYAyL3$}fePCAHu&Y_45Om;`*k^ZpWb68kVycE^hX$+YaR{D16<5Y z5OubvF@o;fbYFQ$LIWD5uP?4Ky2i%OJue0F*)+QUUAL#k=whpm)s(g6<+1m4Y@f4raJ$d`d5F@D!NDO28#PKdK9|&PNk9aLBf{Ww zPxS5u+d8!~i0!HE-PfPe+uXm-DHU)656Ng;pI(OIvWM~Gv0&93?>=7j>^GDBh)5nG zZn<>##3bTeQUXZ}EMYHchZqt`5~P|X38}`2#Tk9s4$Dl4u~5CGLZ-T_zS_i)sQ%XY z+~)7gI``JQ71t+~_Mp#wv;nJ|s)(=Yp86o8Z~M69p<+h(y-hlxg-R0XzH}mSfJO*3 zvUZpi-DMua_fS(kgXsnHo?&gwV_p5x_+-&22?=8i4MTc$wa6nrFM30EvSL0yPpY_eFZSx@_W5-NNX^XpH^Ga%x5$#AtfY zTH*>=l|gBYA;>u~jiyH|se_5fk++({`c}HW+)r2fPJR7&EmA?Wn8yqPO}O;I@1Eq% z#*8lRIx5F@nJL-%%?KwW#xjCwtZt0q-3#c#aVz0x-_%5deg#w%Bl7tAa=f&y3O~Da zI4+!>!9hl1kQv=!*M--5AMFAJETXdn#nFgw-B+J0fAe+hz2mENOU-V^xvD=Uk$s^z z-+=K?J#*A*ySLxM=BSx_`Gy@^8ng29B*XAVjU(1W_dG2T1t6SttR`L%j_av3QYy9d z(pAoF7;&27I5iPI_KjU35Kgtu_T743UbG}RH(0#7g|S+k`NfW@Q+-EwKWF9?7HU}IseCB$mBrt-7T#x&ZK~tN}Iq&OON%MmGkhXpM_feJ0=~w1v7k?}x zqhg$;iQa%j7c1S*9(O3{Uaz!1Bqu;J19bi3BJ3Sev#QcY;=aCdA0zz=L6H-udQ=31 z#KbU2l}+VhUZ`HA)ip^-sz|smq%BCKTl8Ob%36BU`WFq_GpMSynkrhC#fRg<@foHV zNl8X9IN0U%nDruk=TD?Gg*}=b+39+AsthE9(LM@UW(DgbSC%pfHprgWI=pM~BI{$L z7n>LHM_ja#Y6SZll`QAFGl+C7i0tFL6+1wy4?RT}=#45{jL2Ae?Ak}ncl&eMn7@1v z?!RgbCQ*e=Rb`XB$wMGu`q**m;zJ372D9&i1vu2gvC&@=36nvE`4}7NTvc6gMq-(A%}Qb<+f@_0$Mnh^M-1HWzO|7r|!X zgx7689(Fld71CLwQt-;x&cY@C`WTfJl#t|D36pEn{b2q`LKVPGq|R>6AJx!;#+Iu~(L=h9r4`*3H8EV~!M zDXD^GJS>BLBDkWrn&Dno%Z#`)9=dILV|_*ZBS4kCQh1+B9|Ir2yKIEYHN{|m4)2gT+?2)ODczgL1R-RN=izRlA40B z&@ieW?lqFNxVU(fmX;z<@DmZ`O&$+{K4C!YgwotZ9?9yE4A_;#SR;NsmTO+AP!%## z?t7esvZ!h$DatJRjE;|VybW0{yF`oE`s8+#zsXR<6X%LCoQ1o3@#$;jDd{r?f z?bNCgN=;1Yb}}HL!HxmA$Mr}R>=G6qe9@g6ASyZ`kt7NL(NkE z>nC4xKc#+^1*AfcEw(vs`;Q8_p1&a+p!!A77VNJ#&cS(4E<<)1zX2thS;R*g|BAsO zH@os$rIzaJXtUF#``6NUJdt#yXY{NHZtaB${h3Jww>F*UaPg#yQeJs@S#j%`h^FU1 z)m?4q&ScBHc=`zJ%}T;6AMe03>+-OYc5+0RM!GEtcM6#ye;3|Vz2!bwd!Z!3O}$!e z;bmvfW1CcCv++lJsZt4^l2nVK5stQujTV3u%(jM$TP7Mh?`P6(@3xtlxMJoAj7g&- z5hh!2bK-$N`c9UZS6wAt(B5vIR%O-qNSmkCv$yJ_?2eo}BmOp-FvTW2HYO z!9#5>DK0j#p2U!-PfMyN+a{eiHa40-oQi`Zsh_Ti@h&SZBLmjLd2(9K6*~c-^FAI`L=P=-(A5; zpY=!$Ba_6HQYGGMPdfA`VhLuugSZ)qN&DSXJ5ZUbja9)?DLU5#9mt>Tx_t~=;)8!) zF#?y(8|w1P(f!1T-s?@h1Od~j_<`L!S1-N#lwPBHb_}IcD#9Zka>08B?Z>v?dkLOj zu?>IvI2Rj>WyZ+FSRK(MxZ^qQL>No8)Wqu0o}*GyN#QPSCzFKh%tkQ!k^t9&jbe<@ z-vrC#!U*wRi2@bU?n{GzL}DoJS}+pV&K->;#z->BRriy5@3Ha<1q$zNEvP^Q@$?YJ zZMy6HGjQ`+6R>4Z8P?^NV(sp7EGJ&wMK6XXkfI(E9+$LPd*VRtqgf7plo*^P2E)Co zdPEG0VCKYJq4e4Sk~g$KWw;pKWqB+lm{O?=(xWjwJqA;=5-~m_fl=rkyTdvzAnh!! z#uS$d@3Gp~*Y^ZlCGG8eqjUFTo%eOJ^|bZqd~UCwPJ4YLsiLi2yVj^qMNztuo14QG z{9r?H%SOh!Th~gy2yJ;0OG--Q_x8c={M}|8Td7D~T%2J@EF;>VU@2Hgnj;3ImJ$yu zwJ0kqGuO+@%i9@jxp7v0F(2wjdQKkJlt;%(a!)PT+VlPdixaIYF`G#a1~}2DJU3t8 zFd#1I`Ed11#E0P@Xh3dkLI|r!uFB1e3&Qe}21W*Uswk1Y<`$bmLufZb6@pfFNaL=7 z6W5lSW=2MDLnIjp`+*YZ0(Sx9eEhYb5|c^z=WsH<^5r%7%7`RtY;ivJ8YA z71;Zo@9Dbo*}S@MnXt~!*k=dAexMDtK4CyC5JNN(DhQzx-P0CV$ef(^=5&plLASNb z^J_7P0mJ=C*so(inC2E5ox7w5Vp1kX24X`^3!=%$Byb}Cw5k#1RH+gJq9Lq~C^f48 zn8#Qie_w{rsERd-6O>jS@7<7xM5=U+qcr1Vs$h-fWcBYWW|?4wS~;|JT$fvno%FVq zM)j^iVlFtrPa7I%>Uof!C+O#cE_drtQ{IFm+Ef_Z4=z6@k?FB%Z&@vJN~i-(B4JEg z6oSN5=^bvETZZiaMc26dC@pcnZp%2?z1pw2SvDR|ytf-g^=$|v9N%TY$Hips-n?=Y z&YhNK+9^X+z0>M$pX|H5`MnT0ogpARqw!NkRn5hJcpLZ58-cSYW+Ejz6boo?rmc;5 zqp+$DdBg?z+;Yq7v4?hNxpmFhPJ+8k5(+MEv@;MV0#Gp+9Mf7Hi6tHzMd@lL9c`rX zy;*VLNT%ykNY61w}cV@E5^c+QX?6fy_L1d-dly$yNdB*b_uqZb8(1n-sh!7AL$YA zwN4Pw+@k(&r!V8kfOCyiS?-kGd+G)O2X2n!$}$hy>$;>JN;+1u_w9M-YxcgbyXVol zCnY;xJJ(SX4>1ky?eL^}_f%a3hpx6+V{jm(K4CyOTny}@WPy541y zO4d|H7ysJ_dGLungal5s1=X!6=1Q5=2Bh;BNham;QHj{_-cAH@#mi5FR?>6;ZuzN3UHUsar}emW%+x6dA9qMIA$gJr2WO?bau zzTHglEhN%?c`g^Z6SI;{J#9PI?JPDESWsjDsyS&&_dA6XtW70itT@5Z4=&CaJ;Yqr zd)yJWCem}Jq}WHgb^q$eg;+`VzGZ~-&*u%n?dOj|6p0ljShT@k0TZ5Rlx8U}%$(TDbnTrmwW)(h@%z!rc)Ifr`xwQ?FoYc{?pF?!>8gY|| zZn?4*uF#5#76|h@iTs4B&%E@EfAv{RKRhT2fwltx<_u!5BtQ zZz*D+RH}wif-Du)o=T?Pl;%#14mNgpkFDE-$5!W=_fBTOu&GJm7(qNalv_en2;wQN zmQlRinny5JQzW-jNlk=ZJ#Fsl2~SBl$|>o|E3d<@vO0W5Y4$R;odIi&h?_1^$3_KV z%b*rV#r9NQcp5#UsapO0^;LO@rR(|0BjOKZ_UvMo=1*@r90DRdW!oU)Q*_Dcl+<=H zSeE#fnwiwD3JMC$HOr(7bePz7%2q{cWeUc5y7dM3Z`?NwhzmJ|htt@b5yWe>`WNeu z_cs*aE)wt|oJdC~y0|7?ZYE9-B_!GT<(M)&*-XfqHZPh!91p#_1AZLLib77mxcKVI zUAS!8aKux$*AmzIz*A^s`0`C>A-kj+#ascD^g^l#nRK_COtHL@WeTi;`ir0J!t1-r zkU{&92C6L;HMZeeBHUrg&Pti}e6@B*iHs^t8P%NFzA!NfsR>a=${|i>!?wLD+%Q{1 z(~mCVH!De|l&=2>y-C-+nM6z;T};kO2*yKKkHw_nj`YUFNg+Ywm3nj?*4`3MZwP3t z38e|o#Bvorlm_1A#AE-nd>8)b!<}?J6^M&TY)>8%g^@{7R0j%264kUsY>UVa4G2aY ziSMo^Ne(LL9k8~s#i&>9A=9yg1owKb_PV`aDZ8Ku>qQ6&xg_c(ZxaMR*W02h`f5s#-ZCR3DPL&=Vqst__V z!DPB5*^u^{(5V)P#H2NHtEY6kk(<083QzB(m#<>Rdos2!ysvhf2U$)68MqLBBhET# zVnKCUSW`As(~GB_f?(DD8Zrv0$?DySt|Q(8OEM5+f&lZPhaOH)(v#D*dy zQ5{Q>&kL)Y@W|57@YAbi8)Rsi3hgz4O`-T7+T4=u0Fp>Z#IN!<(op@n*VfamZvgT* zfeq)qil+Uz<;+oLGPU_i$jtm#*m~N z0VKE`Tq19K2WhW;&`3wL$}fVunFMzQSNe;|T$E7-EQc%qJLGXsi#j5_MXi(Qj^GwW z97$ylLv&Y&=m-{O-8Z%@WCF^_xUFG()|Bu8F$LOG zPyTo8d8~(&v=^=wK2^aw*8IRt@mPXr1EY5+>D_d?hz6EdB|Oh-x)Wp`R1R-CQ3!-{ zE=!H_$L7*TJo&;741XgRSB#Fvxs#JHnTuSKu-hrlEUfAC*b686k^z`c1h}@-F#Lga3T=knbGz$M%wg*w+0Vvy9kKKFAkWL09B`yO0 zF(VznT#^mHWEz5yz<+LI5xz?sk9Zn*TL$3kiznfUPxr!~6IDK?8gj+^vz56RLRY$X zpEJ=QJxMeSr^yDJL7>UZm4pk7Sdark_r!ecEUm$UrPT+=v2&%^B^}zU^NrK+T;o%qGW%n|%pEjlyqjm-ulC%rswJe# znA9jI!L83LFHQ+0*aD~~B`;NyR7j;WhqfO%bew@bR6S*0qZ(?fcC7ZPQ1?WfV;BuF z8$4EUy`V4>-Xy@K-fCR{8+Y5Rm)3JctM_KI&9M`XjRB(Gt>$Qx3^|Hz7(Ccl8oB0N z_}RMGHGGC!rB$ZJ2YCu-|MNavr^_!{n7G%-a*m_S5z;tOL{1o_JgD zl&L1@n@ECQ!ah5KOypTHK1R*&+07OBpLJzeNE^dTC&pnuH33G`>zKPej^-eT2=ATx zdfHo@KG8G%&C}X?r24qiJ%WW)kxtj-4vUk8fpFMBNOnHx)wT$M2@z?6oIfcY|9D{+ z2B)wJbXx!V@@y=aI>OZFf;l7c%a^t?66PUfQpIU^PBoTq+>I+~RBp;(2cAd+#y?$s z2EOutYcX<27`9VllE&4SufMq&HPoxV=e)`75zCEEa;G`KB2>FN^FBSdEPVcB&RmQc z!qvxOs$S7Ig%qjr<13~kh}VpY7AJYBH$C2lzq2+PNZh7i2+EAL34cuQSU zOo5&;&$%9U?ReEmrU|Zg*ya$s7zu(UzT1e48i_My`=RGmu4Onnm*XUEtTQ)T*VZ`Y#xTWEJ=K;(tHP!96$CA(?v8b0!YO zrL$7;^43y{d+9MN!XHo2xbu9v$_=In-!K#`<#tLsHWgOmcgu4yDk%)vWDL@&+VF$- zx8tL{a{TAH6X>}r$vpG9nTUu~6St%u|5&vh53S5Gy2x5ZEM}cXaq`G6Y`_yY&%k8* zh_~-l8&+(}!)r9~&Pt09O##61Sds(MiMpvHOPoi9XPF}YT`s|0_{()g{r?DfCV^XJ! zB0;63U%^`puRJFEan4*2saRUTCq<2TYS(s*UXp{$Tq;_!LyVMHN#}6~5AxV90;_C%!r6n?UgS`0r$A~(<+7+bftwwl--YLo7Z@le}kh@O;0`iutIO1i&)|2}RG!=l(InhQM{7Ph~t{rbML z?+}CGUT3wruCC5RxYx1Zrv0g;Q+w)E-qf?txX)2N`=(B%+Yd7$leQj1Ly=2^!Avp@ zySLZkJxUF3ylA>19r_n4c1Xt41YZB7;vEH@JLGI@P-w3uMW*LOhdSWuzXd ztV1o4YZbSVT(!zBti!_>jmK?gjc;#(R$>*z@VBpgYHT~Sx+%lQ38Q0i-Yh4Mh%&#s zaw|%{wVa1ao_(Zu_^e{aIOJE{>!gJ9#j`ExCGR+J7B z5g*#ujQo;PJaca{5nBZNy;H{L9vPq))ZQoF$VN`xU_`V180>E*K0@KxE#`v<4<3Ne z!8YtDD#MM_5-@J;Xg*`G#<8F6g}bVSO=`(;T4S>wb?;u@Q8RgO9039jhAi>$C#Fxf&)J6uOz~pIj=k3a)c)I;w9m)QcC+r1P`VofWv#8TBn5)1;15_ zqGN%D%%}6GH&)=MRM9#&J_r}laQ!T*XpN*>dU=4fX&|a-Sz1TLx`^=6qeq)KARm12 zfw6(e&(Al^hyv?{hlMjzx-S&eIIj2tSM@ z9p9cm6uS`68-L(fi4nle2L91;g>@4|f|o3&aHAj=!&=NH?9H zWN3FmY^T3KN-TbT^8$Lb3dRFVwlJDu0E2t^G1dqD!&9~E!;*S@m``16AF51og~lY| zNJ(C($C}6>R=(8HE}*Ja8X1oO8YymPY>%H@G6DBpKEuh*`daGmg%394RWd{A^e#16 z*UMV)@ZAd$MlUW(4Kxn@-xt;)A~6uPxed5@{#bKeoV*Z`H=W)PsH$tl(&8peXY81k zPLsF;8Khaot{wRf&gu>R?(Dd_&$FUbX-|7W5_j*j|voG1~MRl zlr?-xH`G^Bt+Hp+pe{2JJzKxtE!|A8Bs4^jdMd_K4>I*+1zF85CtHT%IpvfzBR0Gl zjZ`iZ1LCCP_1P|0p~nkQ8-r=2Beg->-lM%#&IRbZ#eDx^TtFO2I9Jmp{Wsp*17E6W z-Q=dCmBQFxc5zPMMAPaZ&c*#7Vh(it{ryo~T#Sn^z8J%X4MTi{u_rgrx1*?8ueXN$$`}Xa)`R1E3XU-hlamO9V$jI=h`uA03ELQB3_e;bm zvFjpAGk*2hdLWHPwF>fSs6O%k}rearfPhux{C37Jmj27JoeFzJ~fY<^$!OdPKEIAwzAk{0v+7dVI zx4Y@UnM(EL4k3&T&3iVt^UTqIZOt{uYAE`2ba23v?dOVLj*Mu*gZ`$s;WG;-{Ic5btJc3eQCY#@!c-W8HBXy$5lP4RdO3c(~uI{f}x6ZUl%+r1M-G`-1 zm*UMg-!#mJq$PG-X(3jsRU*uH?b>CI6A}^-ORrclWI!YxlA4;}BRGj6(Xpq(MD3Ew zSxQO@KK=Al^G#oW{dFWJCL%jK8xKA7kdX@Ad+)u-$Z#YhdgfHy;~E+s)+eUwjNXGC*qTWTDs%qlrSU+5BzNv&Ze!0Vs)f5P}LjXItS<9_X-k+GP-X}5FUSb z7e=S8#yywMY)6;Q_&$VlCuU$HnUEJ(Z^!)~Y=c9-7A?}q|swToki$G!mZ0boE*>>w)Vguv= zAdr;|CbQT|+`pO8+Cdk^H=a4hR7d%{r9Nz+`m44>D)4iKy4TG8*i@s_kG*gA_IBrU zd)<9+^Vq%nJ$vka&;3~Uk9Tg%IR0KB5Gu1OTG3qSTNv!i>Gu$xTwji#tS!Z2Y86~O zksc=}Bw~bAv<&m+R6~v&dZN!Bzvf-bAzUsM9J!s{TUbcjhz3Joi@46q%QM8h2v@E0 zi?EN4jXh=Jv}-jzv8UA@#q!v@cdsF~rDC;y{d$h`7}V6%7)C=3)yprxY@`B5GPu^Y z)YMeNgh(ArWu~R28A4t}dvbCzRVneH(+A9#@9g8O6 zvgsqRVtXN0WEbK+`r==&U|d|4Ggwd`QOJ$H;;*8+z=e$8apss*j7^C#Uq|JLNm9^4 zNj}PHRDR>HUdLy7)kvY+V1HgO-rIl|AHIOW31jGnm#b*Z_v`=rkQV4oD5fFa#2E>= zWZ@)Jo@KZ^o8(iB;EQ2ehzPAwk!>TFy_x4`n);7hlf|9WRVe*M?=m^?NTpRKOIkN$EV z!oAWD7c{*)F9ZfA1cWn$%f5g09Hc~t;f{BK+%5t{oYJMXm*> z;Yrb^K-JgFe00FJw~ovw<2P#~{JExQRJ6{b=SY2v4aJRkVms|NeX{YjG10j7oGhF% zBCfZp*6r_+w@j%LMYz#8Y4_#BcPbJyM zG-d`u@(2b*A1Xd76i?qi7dLv)GWNuAf>Nu+=GAS(Gy-Z32r43 z+f?(6V?IeFtRb)Pj&P0a(X`jO!9@;2;+ZQEEot)6CXAxz#bDTL!h21`V`UWa8=_l& zQI#m=y^^~17fndR!tv=yqw^0X%Q%^nZvGJgX(!rb;`gg&kHVGQepu6p-KEv^OjeEU z+_Fn+vAL!hyA;8lZl?{SApCC{#UHy@{d8=Tg^gxq9oxNdH<=m1IhO4{BO(YRqcJow3dv*={rL?{a>3lod|1Ht-JY6(4BRQ^6)2)4WP9sD%#IGgxg(=7cT5}? z9Yg!Ba^3W1jR`w0RHQqzA;Pu4r6bi@Z8{$FVcY56e)nUkDQW9ZN%a@Q;hr=%SXk)T zn>rxS=M0FdVu|ys7f;30=WWD08;TJd9fULz`>Q@J!4Lo6oA~v&FI7%6|J;Am0>se+ z)>nSB44_NDiRoeZHQjVB*;-!c~qDK#;MyL>`(l9HGUCD-X;N9G29Gyyr2kkd&v z>@LM`URsMM-_J!LkzgDR;|6m;?A%?4AKW|z58ZaQK_n6%7FX5dl864yiI0=6-bFm} z;GO3hpXhpbPZ_}4x_kGh3ITp=FHSgB;CY{OgMhbtJY!^F z^XPfnx{VGG#FV%&j2jY#ajCHwNwv~6su+nuQWB%0qI-HmwB_r4s(2QzIryF=)&GXvkRucoKl@g5ol;RvD{#DIQ-C33|+oQM?LH>S} zw)i1BB!Hd>18Bq?NDov&X5mW5!IY|Z2Bn!3c+uc}Qxoq`Z$m)cSl>lVhq`?o8Riv@ z`;49DO<586DT5f!7@a_GnIXMhwH|q&+A8Kl^ogG_?(31)bh%IsSK~CG%OxGZ0O1I& z7kF|yKwK+y+viCgXaFS-VL&vGQa_Rt$&`TTSMdMxMbj`UEfx!Y`z|)*)na^FI5u$=ZP7!oVL!vf|25IJ$MKxp>-9Hl)9hEr!)uxh zyvc!pi;g`paP$G(OqU`_WeIMrvP% z2yx=QrhWU-M&?AZitK8s%J!uPFn`Y0BB;Zt-W5X|1`*v6w4;-{X)IMqwaNY>oP@Us zJNM*Hbh}^g>0`676+HC3RGuXeltKFThJ7eue?Ol=eIJ`cTg(Yb7?B!92Ge{5D?ng~3pNfJ7Q`yeHP5!;VDJDoa&Oh4cUl>{Zf zgp$uelI40>(s!YQg!n{1E^4Gj#SF*&(yPI&r&`?;!ifD^nX|D3`qqoe%X#v*LZ zp~?giV+v!5NO!xTZ9fW28sRr24DZ}`0nVJ5+0Iyq`MBjr|H3!98;_f>q zn?QUwzw7|cm1n$ofWZaznpG7H&p#g`q?hc0v? zneTK?f>#K;UZ5n5JXU;w6ZvKb57GcxE{FtYe(K&zmf#kiF$@F`L_En=cr2L(Z6jQ^ z2kvz;UXzHEwBpuIsJjsXBAm4e9$H0p(P!>O9CqAZ2H)MAME7Bp=yc+=q2r>NOgXbV z`;O~^mrX|I;dXAX>!A+fUa)SzW4ecVL@?uJ`%T<(^w^=PJNukm&K}~4oRc`U=&@J! zYQb6^Ka~|>4O^>XRYpd8ThjrI5A(y9C&yvYtkilPArm!WVM5Q_scLw?5m2W(CnC(A|escYOn?k4jxBAt4^;&W(rfp;o@9 zr`~powPwQwk8Qcd*qT>|PzDY*eex8G>-uB;e+;<8z+)H?Rm$R*+pd^{4LgeQyT5M0 zurZO?)p7utWJ135qqopTas8cF&Nht1J{~7hxA>P|J0DlhAA_I&b2VNjBY+5hjHYT< zNks$x^SRF`#rPClD#R#8Mi1qxVHi7+Cir^VbkxypW=;*&A*#7)HYYM1#b-4PN*OR| zZxL52xZ?QZ>n7tXj4>i(b?GI_=W||3HLm&53s}3Q4CANt zd}SM&Tb*yL@_Gx6>pg~Wa=lRR7M1Mmb;87ZCLa%tO$R4DW7uJC9_PnZ){)8+LhVfK zAE&fFxotqBy6Wxb=On7Z{O*LkokLEdgdWMdPh+~G4cT#RD9||8Dubs~uB$C(b0*Pa zRR|fZ<1r!Xm-?hu#j2iCjuo}^ip070*35AH7gelgj!D1}1|hV`N~+X%2gLko8zwjq zcU3nI!t9hF+!$F(Jtxu~9%LV?t}3ZhWo;{R7yx%gfInh+ttTu~^lckk4kDVW_6sui zBm7_`=?sq|Nr|Tq8Huw$RqU759inQdBd~J&q`i7*7h6siP4;RfPvT`B^|gg?gnYD8 z_2X|`f?dURc$L1T;{X6a07*naRAYHQvKTEjhr~mAMhNcu@jIxbC#&z?d^S7AV2BBk z9`NjO>G%hAidSsP!@u6&ia&jnC)uuHMurkuN|abd4_XBrOpPvrPn<0mAuyZ)qXLJ7 zns=A-ne0-!i)`eOiw(jaw|nj7nTE zo__cmL~(UV0e)E^=ArsPzz`|IV&C5pgYFfJZ5#!%HLA`*lB zxGG0Uob8`StGkgPak1dQ{x%es22iELaXH(U$lJV|oxs^2x#sOdto@)b7d7Rk3QLSZ^m)tcu8dD8|R5Gn27kB2}#1V|*~b7VLx{ds9-?ldw8c*(QDw zgQEJkH6gN*Axrl${)NQ{M|z-_uj$)Ky1Jj?>w_pcuIG8r`5fc-CLTX@klwTQ)x*~? zAd-F^4e|9dFjNy(#aB}F{ZzmA?U2m%eoY4E8$F5v5r~C+M7)PnxB0I>x)xV_?-{K4 zxQIdh!m*3Y!O*NQJn*}f*pgR)pMU)Vq`17fh!K$u$=Oq~u#nr=uPnvt%?0>y!yY`d zr39#@!KW~#q#RN_<$L6+v67c6DFrcxoLYH|3c#0UrD4J349uO7fee?&EXz29P! zB&AbC>1LN$xCd9PhF6CcQE)-riU zlR0&iwV{IFWUcWRM^CB8Eg(Q5DvA+IV(X5eUcZ3n(voLP;*aEId@8_Jnn2{k0#`et zS-y}5{c5JLv8k;ucb-0pLpEh=)k>t7|F${6mbm8oJht>4{X(xerpS5G=fr@CP(=}F z^%C#&q^W$meCO5UJKo&s66#n);TUi?8gZfdb_Vaa%@k^eetE%$~(?48cJrOV|u)xAsF4!%lvkmL}PY! zoa#Q3@n$BvF9Pp$xE}qO1k_=S!!EfhYD>Zi47qGgJkZqj_EBT_L|R$ z!_=3NtXol)Mz!rn_whf!W^d0sUGFVNTo>?$^L2k!2&Qqa&I=I_>X)SRpHR5<{)?%- zYjj0TIC$XOp0SoZw*Z6UYqW4Ww%Q zyU)!P{<(5sO#%6}W3gZ4mg4ebYj*o?!qmw_GiSl6SCb`sQY8NF>~}!rl>fpJ*4?2U z+;NzmLEuj0fi5A7#FdW*{gXfj>hvT;>E&3>IIG%{)AF&si&XY-%M(k`tJJ8^E2naI ze@2*T{9cdOW54)=i1X;6vLi)BWtypfFl4ZfynwjUU8?_YB3^_!3r=u)f&tr-#MuPZ z$wQC+NtRTl+;c0qKmxf+6uvxfMx<)nBSxDme#ndW{8YqYFDVq|M~+JOLT`UeR_g3P zbrvF-<#?J{U+_YVcjB&|OrHQ!4`PMxSmWRDi+MtXJOA}nb!OBhpZtWp4-t6tCT3Gk z=Sxtk25~{DkW?0ZYTSuzzWLVJhc7JPcgQl=_+{7dQ1b#3-S=W2);YndhsSj{@fWk& zgJKA(u9ug;50Of2Rs10qzqjYMMe`u#F;?&QO&)7cFE1n(-d*`N&6x2tID3p(ISX-9 z@8;ancFLk5VYmC&qP=q2=mpD^?by{Wb=c6Xox^-)OFJ02^D6=eJpLT#xVyUD>sd)0 zZEzO!ni<3~lS&^^3}E;&)RDKmIPAo7*k;dEB?aT_C4|Vv>kIIBs%Eyeh7J7(4}5#-$P+N`}{yN@jCWN@j~; zB6duCz*(RgRaRKfwQ`SE6vJIYVLvm|zt5I{Wjj(60zvbSl4_2vQl>y}`cXZg0fs?LPs5CaZGBOP6?ZGF0W!gE!)Q7ES z8vZBIW<^vUW*?VYNG3|uyNlCIKYT#h5zog}ucxY^p?-(WNJb_)TC{Q6Z?c#Nyd#BY zDGMWN2zRXKYN|j-Seibfg673CCO4pk0;5Xr>=Rc*b<~+RmVDsYUgTc~pN2&DjlA}2 z2GjTV6m8c?o4y4T%pzu66zE`0c&7_~U&*0p~#nEz7Mu zWQ&ooOOLbMS_C{%Z7NP;H%uNXl;+e6llX-tlBZ<^-i{Z3BNxU`m;c$F?)NdRJQpKw zdu5IT(#fMS28sH-cP+a{_qi}!!N=R~^WtM(@vp+)Bq?ZVr$yr+-kAYMWQ z0%7pO8mB}fC(7*cG%c*jw9Xd4;@h{_`t?-fPn~BYQlE=hU^jziHK;l;{wR&Qmk@f4 zsoUHBc)@+vnjVcq%)EN|>qt>cE5SlA*LUw#>)EiF;ylCd;5s7uWY#Kv?cr(#Ip5%j zf7pWB=L;`+l`6c<%ya|!yqMV~BxwFiK{vT<9HaK>6we2psU&z0vr1S`Q++*8W3bQ# zD`1?MCNf+fv1i@M(KV*ZpUpAZdP4)m6L76VJ)mjC{Gg!u&1;^B9DW>~Hrx}6%e-3X zA$Bs4ZG4P!e`uS&8F`y;&1Q*_pmx=i@BUp(xR&}v`s%ibXiRYY+L5usV zs^-6xL)jw4Nns|ihl{@ceqO+JTKl~X6AKHKq7s|(w|C#WxS=Pw#5UeT>Ac=&e}r7N zQuHn7e%Pc7yD*^2$y`|j z@=Rz`ZB^TcG`HzuHp5vU?Lp0I@DK6s6Ot++Z*q=#sRcN>NNVk+kK&w ze%m$@*QrosAds|?-%;8zIu_V#gUhn6^XZD@%SJM%Uk%g|9aYLb!lU`^H?~41xe(I_ zWg>x|#@pW=~n$qLd zoS40yS?`n%XVMorwb1mezCPu*dU_Op3o~jpm|kBmU$r5>rW8lS#|zm+DeD`GLPT7P zETnrXm3_bdxRWDl)BoW&wZunO{rfUAsQ53siZVb|C9!~t~U#%5-Ksb8$eOHNj0iP7Z=uW9lbaz`hp*AIG)1#6GgZtB*u-mp=6 z-#r1|UKQO4+LHvPRI5??mVnQW(pj2l1jmb!U)?{D)Ng34!BA-bY$^jXMLDeW5fN%MkyE6h^9%rK;>si>$k;)`X8pkr&ZegDgoqCfm_P(x-YkeO405lgnG zL0$A`KasF#d_+V?VZ7uAafQg$ST2&R{+9Y0sj zYcQu)h`hoKEZQ3@^jutA5@g!R8}gp+x3N4MZ*_G2HWQVOkGlz}(v4+1myZU2&oegt zno*gVeD`_J2X#GbKd2xt@2yuVkBUz!Tserj2|v2>jENZD%FkS+Y>g2=POg*xi++&o zq>eb9RxvrC-PksJYM6NCcQICzrMR+aDJk@zM0^~4&h6Z z*!Zie+$jSZsL8LCLnLKPcfIG%;4JpGA-TfReTbQx@?K;C#kOH ziqeLon&ZT!L6PljtHp!oskODW2Rya$gZ!!g%)Y#hV`0K|yubys@s*7X4$9fs6dS%K zv-qq28xuVM*D1L9Qb%8Ws%U#^1@&IsRVP5*yE^4WHGZ!$7>OqGUp_EwqKdY*hW zM)%pUig>Auk;0q$uw#J^t(RZou*i*7SZ3CU)9>7J25U#=0=BmgnlG#D)Za}r?sBXK zJv*exwo>_7@4(^c;(&-E!5T4oy)Ys**1(RTt)E=Fs?cC1%|mw?Qr6Dq{Pm!j;K$ff z9PNychUmYLQ|$n1rSW`MQ~~IqNA#f1AQ@n`1WDU&hOj(CX*-jOGyxM|p#4V^SotZ= z)LA4;ZU+ird?3fa!iJtE=dGR~AxM;s#f}_&<)>4fWYH4SixBU#mBWil5*MG-dtJ`E z({f2A^htmy1smc&%0qLt0+VVx%Q=!hN)$7mi)p=D_A9{yzZXqi^E-d~e58(c&i3u@ ziT_TGZ%+S50vtpa_>ql`6=U6qq@lsQA*FFTbbfyNgOelY^t)d-KO3YPJXYq7t}xbh z5Hv#ah9LkT4m!qMvWjiq)zd@vl|=IXAXCOxPQRLK?&k3CtLBm0-y?oUH_L9rXG1yr zF3Bf^0lzM$jSq|&J`Z7FHFpI)a(?!S{*E%}WW~_ICe(#Q6hW_ZFN}YF@>oef3OMr1 z4Qpb2_)&}xQkw7lZApa2Ot+d{z3t+}w;W#Btj^1I9W0RwGCL19R#axV9_ zh?Goyjv!BdV|ptbb-}vR{Hed)6HgB-PjeM|-#&awDy|S;ZTA;|7e)oXT>)V!qDpD_ znD`Uj>V48ABP$V=5mML^_mY zz7CaUTYh@H`i1t``1ALl<+I$YS*qn@EbdFHmFuvPOhi9PrXeni2izxn+~ITWqdhWl zd1Cu#+r{^gOuRm$(t1e{C-tdZ2%3vU#RiMSuFA5|v1C%vr8Ez!2RM)}{p(aG zJT8!wrL70j?@1f&!GDQkMf0S<-=DbJSG^3oEev?4+hKo3Tb0j7<-I8)W?=z}_NE^H zDIE6>kA!UILo=j;76E=wh<`aBm9O#YmX|L5n}{D_7hI7&^ZL%L)a{?lMtOSndu$lHo4NjNIYK8vN~^* zr7*eo>?NEfg;J|LZ&^z=(zW~*$*KucU_W|YGzqV=#F7G{TW)@@{oTJ zaJAdrc6f)Cg&p%5@{V}E#E9$|1s;ErnHUNaj9K9uVO`dix?dyAfmfRTo?qFR8ZncU zMb!E1jKf3(BM2xb)w2Zgxs?5ywk^JNh1ZH5wN4!8Z!q>buM{T2#4nW^u&l+=2Kk7C zB4z1l4gUFP(MLc8-ol4MO!JX^kV}EyMv}PEciXzV#YfA$bAp(4} zVRG5@G5)Q7LsqmKw$4C|wG~&U`E(Cv)VyXvt6l^+7CJFU(5qjMV;1xE4jRI=ceBWr z&wc-)NGfjizF8yKrW*^o$mtcGaRN~MIXe71s73BLiXw(KB|U;|Euc?8SAd8UWY9D7 z*v}=N_8B?Kc51m0jZ7>h_o~?_2z)P2TM@JFF~FWGk`dm$Ht*ZCuhjAxglmiJw0q|C z3f1s9)eeQX4(t7k7Sb+Q{u>r}RRpjmd{95}F~wgpoizI*<9K^xE;^E91Wl6GmB6Ck z87+v>k;(5Xjn5Y#6NVlTo`9eY3UhdelMPWtpAfno!o0WN5#y!@>V4GRGFAM=;^^mK z#%X%Imx9ge@LOcg z-mn==ZFckj!uz(_h4U&>nt{gy6tQi2z4;ym`yX8*X^$%d6TUu+wS#f`de#pGd#tJ$ zo-y#Cc-%uzi`=aC-(r_6 zmswObeP^=mUn%R($@@Hr3S+1%YHZnPnm6@FL4f^$Wz<6L7bGRktcM)5_JC_Sx-9yY zM|1qt729~QUi$mhd!K1TJD+)@=k!*)R)5t2L_zZm9e0r(J;eFX)_gkTiHFw1N$Wi! zv~h7)FrtL{xK0jEaM>#ipm5nIddQGT=a6W+CrEa-+#xjiD-T?Nc2dmqpEI`UecVuw7LJne| z8hNqqQbcngx%bqkB)Ja7|3d7TE1C5UgVV_QLGR0rKAIN2DG^n(Xsw_?=aqK`zd1wHJ&XX*0Om>kX1tB zPfV!2X#rln(lF0FZd-2aT#>>ZD!}=^bo8`y{89zOb(TH_hbV|K1dWhgu&t8EhSmt} z9n6+ZqHVmlyOLF!CL?vaWEvzu4*_wO@LK1yOwOD`Fm!%#D>pctIlsv_TVw|q{f$tH zMg#}JF7#nrNMNhK14~Vy9yQ~?Uh)Xv9a!sTMD)J4(yE-l$2_YQlixrwwE%AAE7#rP zG+)m{1wfUq8E~7LpSjRU22kKM_O43FjDgd}mD;9P#06ZC-9&0e3qG&yj=Q!LjVr8T zWx1U4IC;iD@3vQ7dOI6kKlYiFdZE!NODsJ}@Rod+jiv|u@0N1MhzhehSx5AK*iPTpzDm(I-QCP?!~&bgd^Nm1FO@nd(W-`Cd>n1 z1_SyZls}sS5SNS1hmT3S~IYm@tUpXu-H$jKLr?ZSbOjZeVr& z{!I1H{iuHDl0-kCEL#A@Mrd^m!w=s)y+d^Ytu69P0iQM~|knk9dby_TPS z+=e~cnMp^H3>wl<{Xh&Xos-B=t=-*>z}h5GIL^c3)(dv!m1Ds0@2L}VA5?lXHv;{e zPB$bcZhD%OcKz=S@D4cEBli}%?FB{Z6Jq}yYpV%vx?jbXa{d9JQ)9nrnfX0TN0n}q zKj*V8wzBKwdpnVLSVT^rD|}0E({#|d@XtZhFZ1Ch@2U9#3A+zpP_APWL&r0{9Z$QR zC$Fb5F}?00yM1#CBH+23{@J#5^^}}t4mTR_aH9k9mWSGI0;a zKUU0G$^)E&E?)V+6eHeY)o-2%pNOOH{k;5dVwaBXmyQ~u zWzo>YyjtT}mqoX6oBwvS6>iX&kfaeJMfe}20BT$K@MKtD-b6A8Uu3lE4d32})rBn#p_Mw$Go zFMC)d$42M@i(d79aG~~_Kc8Itc&)PP&*P<^DA*IK4Ic! z!E{>Y?kMroZ}LB6>NkqV9TW z1}$R1hcfkY6+cBoWugRajI=o-6f{B$!dVmhH&RY!L1clqM$O^`rz?4YC(wm*25_%=(2^@Lj3O|vO_LVHfo zj#a0M0H*y5VEU3vb!WsK-Yemg9^!z|_f>L=%JH_LwP>b6uV&a%JA<@w?m z0UfP4uJZ3e3h>K^yQ@2ody~=W8keI!LJ`HYARlu6E@@fWGP_pAIc-TB#Z}P`BZhTrlT?TC`-X)OUZQk(Q37?7XAkbs>Oe?#W+L*k-b^P({^ zeg)#NA*n+^j;|nCvS;8ycdCgx^=hsF0)P38O?(OKv!p<3qTJSWpc(-nimAwZNZ$>ta*9G*CT))|M7P8F0SSxc`*6#rEy<(cRqc7XYFV`Xf4dW;hN z_J@V`gA%(Qmz8zL+taPVrB^xLHS`}Bo5pZhngX?f?e^r(G0>;G6)9!G!{1lSo%^{y za2ST$o@_LlgTZg#h85y_F6fyM%&kU#C`&d02?6N75}+^5gSI#-@uYR#(8gk-*D?2z zp2Jh{1*k8FgBv45AIcZf98a9J#U z#kz1*cUj4opqfcE^O00rqm zq7C2*3-!3vc|Kt?&Wj5fZ#udJyaAT`F~~#WU3u%>($VD#uqS@5m#4w^KYj?ng2k~{ zHFdpaO&|ci2=G}A!p7P!to)Du#Ys7JNo`|7uF=Ngurn*^Sn5xdpZ>O5+^~TEi*Bz1 zrHK+=IVb;!jFhGeOdzHTrVkcn0?2^r3D8*^`}i}i&O@Ougl=C1SW8V|2CU{v3#acs z=R|`-5_*p>%zxux7(IjA+S=ZJU%BNZ^E|ogn4_)}0NB8NheDOHEh0O}U_;q-?Eb|_ z7wVv_mQF8@Ad1Fg<4%!m!@RbeKc8DPpKYSUVxI0tp4=yx1e_D-F#cMl%g9tYM<5p? zp`*}yeiR@twUQ;_hbJDPehiOf&N!TNdI9NkGO1W@|215Vs$(*XX-*7ghx05n^x*W3 za}Cpo-e|LX%l2Lo`Lw`kAcT#2z4Iz##J{Bar7Wc~aVAwCO+YaD8dLle3g7YLlF>(G z=56Mofb@SW(#@b%nl@46F2ZEBQQV)#@pf!>{Zt|9x==<-n8PSzHC>*rSIA8NMl@

Rsn2j=*s)jJtJG7pM!-Q=}c%-QI>UMD;A>3;kjNIPfb9zAJ~4D@~5@GBE%g zhj9>4vWW@L2#S32Be0%}f`xZkeWIQ4UEjzI%yu(a3!b3)7&1k2Q?bulQr)IVD{HP5 zK+x>diEFxA@*XAUySm9KW8@ngcJ5umL#3v+$hh$LK}p^J4C-I*#kaJhq#kmrOo~Gq zBd&Y%_1hY@KKcL1L@|m?ph2SAE*q+wML1n%O#QyYI%GT}2t%(c5d8MSIh!{lXK{og zDCsCkYW1{pd)WE(tQHt6X4{^Ab9xvVAJE+|MeCr0$wymen-SRopaTU{-X)6nF@JR^ ztYWwo*u!9U>21}b38T|F)k-{1d&MWI-Cm*!oH>I5N_Eq{_*#|6lS8xf zD=iOvmOu8lqT}LYaYj~ruN@ zyDt44_@cc)Unn(=%D7>biu+>kTrNi!stqc8yYev!MpK^sOt3#YeYi$=!sPCm-j{wq6jcA{3#?M@NN+(&rEFh9K=kVB2Y zC2!ro@}W#m#L7u&GwaTpOYK}TVn%+(xRU!2(_r$AGmgm#ztx%?^+D|g zb5DaMRTMJ(7qo`2;><+@cAR0USeLHRoSr8zbra7W>6h*n?8zm)|QUL z^>OnR`=QX~dD?r=IqvVaBNuCp;q~CKWVb7B#t2{ndH(~k#QIq-ZVs_K* z^p1Z%YVSHNYtA!3GxtnSF;o~T(!Kce9Kxu>A7=~jRzXB(WPFM00zRNBIHJYrGB`v= zADGX|w<&9iPM@2TGOq)Fu0Ve+xejtdhw@NsoL-ZZ>N*R$OkaxXR|q36cw8%@i5Fm` zSKZKteWAEU+!$OfN?}NHsSyvoIaHHUi*SVMh_G<=8ZRab$BXkaN5l(+4=R~Xf3}FS ze3yZAcs&s9%E~@)8GTEzD!;ZpnG0c#hnYKcLOCQ)+W`zOsFxkgmS20>*!A4O+Ea)@{|@SYa4)bi#F z*t_R~HO-D-Gu5Rd;AKw(E8`^?V&kH5$N1u|%Z2xSkp-;Gl@9SA9XsM*I=Ra#)B1;$ zpAwndQ^DYyJ5+A`@6IU!V4PrByfFZxb8mQ$jRiG5mTLxlI93=O6f#XRi5E!By*IIQ z`Hjk1Mni{QmrNHEV4PI4xS|r5e$WB(3bpm_K0Q!VW7t=#oPF?U>{CdgOs(u}w z8n%(w6WxlbYy(be(@lAC4Pot4H#{fs9QJR%YokVKRs+yh>RZE!Ky5x|p#|kam zk3cW$9z?m#OovS3$+QKs&b-%7^~nDKg6bzbeaYefu~$i4V#hFfkskfzEK8rUTzkl0 zF0YjUp}EES&7ox8(if0 z8disSm>Fb)i79)Dot{rnlMY%}*iP%qoGC?6OpyQNu1W!X*S7eCQya~eTVG6cU$$1sO#}mlY7z^Q72*)u_ zf0SemHX{#WW+zFNs}^N$cnC|Nak!2JuP-0_*-E6({66$(GaXVOdvE=pLWJMiCuhDk zpx<_yVoJ8(R})8IKx|J%4@GiZ;xyh*8%BJ&urp{%UfDu*Rq#jGErBZ1mHI>L{C~Hf z4zMp-$2WQGPy6^awO2ZJQ%ZmstF*ZtPe%!E2|m?+(map+!HaI4LyIx9a@{LLkZB1< zt@hQN@$xAt1Xn!V;P}hwTwnYp;D<6*dTX$sg1Buc2QW%8o%be6zqM-y5VcKfLySRq zh<%lF*Xg|m?dd8ivdE!T>mn!PI35TtkKvt`F5q-IqOhA(@Jk;MQ}%3bJT)sJBh)d8 zsYdW_ggwfAXeRp@Ae&wUC2C;7WKw-acQ;g)b&`vA6xUi{}i zOu#0c>yFEPMgTT1qya)E6`GIyR_?@U*U-M>mAv3E>8sQMhDzP^Yqm;^c}n$5B|k?$ zROi6(Sqmj=(F&>QYw}*)pcPW)10D0G$tcUCF&#f6{WF6wdP|g_${T(h^EKe%Vnz&X zP+r{;^D2UW1H%aq33LL^0qbKLzE!mhsJymHJ2$%-(DxUozo+leg#q``>Ex~cWWLs* z9ffDR4!mkJU;>05{}pPVLtdhMPqC<8Hg9CytfDy zCV5H)=Mne1h=&&=Sa^hS{w&U`wfkso(Dah8eWpmsd;2PWRb$16K>D6s|76G<#qRAe zEKnj<+_RIhqf*I>at&K@H}NbwcbV*6F(zz=NY8YMt0;t0XF{k1_`KW;mDCKm@V2=n z_C8L8-=XJkKm+lwF!{(RIeM{2YEgET6O9PNo-Z!2WuNWlr#CU`ebiPEj5nsFZ$shW z(XK#Fs*&9j;ED0qPq%+0LqMY)RPRQ=vf}h4b+GU5%c&5iC$$B7n^vJDmbiBSOaDQa zDkis-@#*izsyc;tM{D*6XJez>x%#)`Z^vi)l1yh`;_$xSC$%BAt~R^fQIgt2f{`jQ z6u*U4iB9iZ{^hDN0AB{+u&X4jpLy=ai5;@4Xq{N$C>Ntduqr0>;`3pzwz?v+?-1Qo zcw$nYnu+u+I?wbv)LaOYXyWtGK#OZ#_iau4A{)Kao z<$$@%czj16$NQ>tsBjCm?n3nC!NhuLM8Un4W4vxxwZ&{ZgTtcb{QKvC!4b^u4h`9# zrDO$lHN)>MfsiXQ# zX&$>>W0@RV-^3*pImRrzSX!24$K*|N#Ou%z)I=UF&L}by{(@Tx@Xh^ZE6p!kAUMQI zp=W+&fvEr@uGx>gX4zp>S4qabJ~4Qdmy%ag9jCsjBNw;)7&(+8zo*pTfG+d&ef-lh z7+&^S*y-;BU93)|9iA}iDR*ml@>7DtbwGSD_NG8bPU}w$>+7M&?+cWexoQ^<8#oEm&gmobvHu1rm6`b7p%$DK*(XNofAQBV$P=COxlKZNYf&ke0It1O!S{M% zejeLxO9^rT^v(P{`!Az9=Be<%ywGHG;*nr`v&}VAc)v|0w_K+1V@ccj#9tOv_|7GX zCCS997X*H?0Ny=Pc>|9RasaRtwVTrKgC-Vv^apN8)2uiSFasD%`w?bv1u6nX`4 zznj3lvCgRFyF+cmMZK|zz1uA2Nv*j2m%X7@pOhg!g2V+}QspOk3}M}`Zqe2)X%NY4 z*13HGB)YN>(u!{#i4wq_DUrng=R<*L-1Uu(g7K?cL{wlP%i3M1NaJR@{_%L20o=SR znkdM8szQ(HEbuD{1|7JyiP2n%S{B`g#~6Hh0Hg(mrJEyMNMcGq*ofvqV*fFp3abn0;F$kF&_f-49F%K`g-ekg-qg4+eo%GgW0 zp=4$u0ofoy2g$~CkOu1sze;AA-wf4zl&7}QkLRB!q2DX|XBpwkBi2<98V}^@uY3a- za^XQ|Ob~>dLjB;^f3&%rUgf$;KnGnUY9cNZ!law&<&(qD_QdZsqS1*Ny0m6NWV@J* z9ltCs*|v^=5Px~#IgUlfk9xuDl1WJ^gs?CMIOKIdydo5(xxlKk3#WFISbYKCbO~lK3#X)cwc+fq7Bh> z`Q~vt5oQ@to&(CL7=;_+hH$InL8r+)ET`IPU6c2}p)({mxo*LWC+tH^vyoNT2xV;z zp+nf~zly|3fiYMOL&+#8f<)zM$NxSfZ5(N(^Qk=zR=93o3!_>TA^Qr#ySeQr>%4y6 zudioXwz(rTijd86QGU4>8;J53?Mh@ z2O!DvrarA24mhPz<2eSPU@Ze^Bd~f%Sb(^TC9wJ`CLUBDg^%wI<2%(p(x_ww#X$iCDYCyfL4ayw?Jb zVIK-PyyDK~P81r5cI#y=dY(snpYpECWLIU>AG8kbX*%m0$YaG^3^@;yfXZoeXM!c? z|1(4&#jW!EV>s(s>v;!WrAnBh(zCdK7|1y11wJ^Xr7F!`0NZgR z#S{&x44lyRLmKju7=DY&{VT%VGF5A_wF&uu{7r8K_NY>H(%B}9(9n}GtYt>w698P}P}X#O_aGfTt^z zHwM8V7y{-DZ7OuYf#07G?+n$*2l|6C{Gc#kUAidPSv>72b!5C}1}X~*=_3v}jSiIp zk6?rY<`Eywg>{vvUhxAgsVi|T=3o3(N+4G|1QwMNf?mU&)y|uQw8^g62hQ;1?s2-{ z-J*4xyd_~R?TAOjLnxqMvsZnF4h~@ESd|PmUeq5d;Ms_7tSODPXWYczzG=cv_jZvQ z{gmJAbLf`0KT~jM`HO*ouhZ>|KS(Ts%{tL70{bdvn-2AuMF%TR+!zOODsD;r_*E|0 zdFVrs|CfI<;5htSk)spEK)EA#_FL(8cbv60Y1Fe8Bope)Zs?IbFoEqwdMUt|BlbI#7xVfGsZw$T$k8h|3fWYJQ zv|NsI1aNcuwzXKV@#F@ipsSK zur5xR`Kym|9g7hpH`+rE!T~ZnJh2Qs2;S>3(va#Kc*lh1b67#Fz3u1#&zT;GIigz+ z)!@&F>b6r@#{2Kj0XKNdx}kKUI8_H}G6TR51AvzIVxHQd5~VCu)9eviMf$0U_?LbC z*EAxd(Fd{36}RF!upD^b_tFZJ6N~@s88kguR>z(gB5|y}(ZL>La}h zv7?ZQrVm33JReWjN`Z@bBW$|PLGYf-arhN_+Yim-F8?AzT~ zY80gbrNX~ltfTzb&oZ^Ji=3J3SH=Dql!}=-%~e+?`2O?wbQxaRo4xae9nzDf;*UDw zsREIUfsILV*n4>&CHQGsO4E==4+Z>{&}BA9`>ZwF_Z|#P9+Yy#GF4%?8T0(&M)G`u zSQJADa&8nSIV;WxXC~A$jS#O~M~Iqw*fkd_1(Lmx#WXt{ z6?Y!eUp2yRt32rnWtCq=P;m3FTjC)un%xW1z`ycwB31sYmr(BzBh2tr4fN~Xa#eTd$aJv%RC1NcMFzUtgA&(FDWmXiKlv%1zyzL)xsr_D` zBMf-&{Pj~~2UG7Wv&x%H!r`$eONmK^2;l(*PjvlTX&)0t`HXse{~g>@{F6kfTa~r~ zP+XmOvNS3yDSI>JLlBC8q2yc7eje>h&%GJ#Si}gRRTzqLC?Eu0$pkhHD?;ocTeKYV zhN}oT#d(?+?N!mOB(YKp19N`yGph*c)ptFC!t*Lr-y&TnHSY?zHzj5tSX0H``mtjF zp{0U^@Z>Q9m^bN1!}nIQ&B*`ec{B6pW6zU9(|RoUuJ|+J|DYUMq`FKJULuSSsSKmW6f#f~e zd?|n9G9*Gj-s^EHe`B)o3|@|1f{zf$6?sw<##juwZh_VebgRMVDuMoBmKR8j;1|Zc zJpn9n@!=ogmXgEY_TycGYNAJ!0KJ+qb8PoXFUN11UMUWCMFhxHPe9U*CQ?GsO(~ZSG z2|CKZmChW+lQ)z*DC9VP9{i>y|5k_E(S@+$NYz7f986|a(}J)4lz+>YjY5Tdv!gL^ zPq-V#MU7s(PrE_4)X$|;{)INDUxLryf7FDj^+S!5Sxmi+@@QC`!vEM$;Q=QqCtHzk zIQcM>lQ0#+8#roON}}r>p56N1>`ZCSJnQjnfJx(7NqK) z&NY8&YFGSc#&Binjp+6pM+aEa#h;dry-edllaivAeR-2OEoGDsCUE2fD!`k6U<7rM zl@R-9g!sP*E!kUE@Li8ao)B@ARL8NGaMLDp$?`(Wn$|r|$KJuIhHAU&i_)0~`W|{1 zv|hwjIR0(3xD5Hr*tU+f%Nh6VR9nNtB-@R%O37sM_1gztF4S>9l{euv^DuPbyma~9 z5{E*mR%SH%;_)q!(bufM3=~g}_|fd)zl-p+PxAb9vKaapbj)98BQSZF^{bTq!Q5ObIwaatz&bITm7 zT&`}=AZhL4RM8$#Jf;5Yt@S2#xs^u3wA^36i%hkz8UO?Zjy3&!-)HU2EF$?LIktGp z6HP%hG;ux^d6~B~q5;wX>zwQ#&?1q_c4jKB;@sb}nU@WQ6T?^YnRhFWGz+kW7RZ_L zwD)Q9>xt9>HzL^#?N8sq!rnfG40JA-+L*Q|&4vVE*&2_O+cW4qwm+}~9_xOGwGKJ> zorFqE-8$DdTxpGv9qM0OjD9{`=u|$+ABgn2)|#bZGix~3AA*(nBiyO=OTSA(R5J5V z-_7WYOTC2@EX$>cOuk+Zi;DLp^xz!td33fRnLxN)JZjm*6mfnr&S|^7H+P8IjY5u1 z=dZ6DxoEb?71Qj^ccT6UQt^PeRr1421HW!Cb>PqtJ9v@xYlAdHIbeKGvrVPPstUhe z<^345n{>0#)So26p-R;uqmhJL!ZV&)@ZX=Vhb#r1ZSV|(xwmTAp>*nYOD14a2Z5~V z2={C|LE~hu^JSL1*WcIb_EpD=?oF7K{B57uvVLzcPT%Fk8*KOfGBV{y*P zyL*4fOy0pv+Iz?S`+sDW8~0xG=M7|OTa+_lTH8Hv3t|whJ0hIc4D8Inuqo$k`RU!X zt2NmC(63F9Z`n%h2j{>%3oeH44TW&Zf~4Ug%tOpuKTf~TZp52DYl|7fO5v*-!S%u` z&6A0{>4i{yNUb40!wYtO)w5bXajUwy;K4tnA0ATQAySya?%uxp^-_)TUiEAZN+B*s zN7kcl`BC(I^`nAPV$}~qhOS8ilkP^xO9fkgh@;1^VzAG#ROO^|{JpeBW8;C;&$eob zKcck$X$_KFDbtf2MDUE_&AUioH!~fg_ZY1wC?bP_j(&2j3AV8^RXU!RpnTiryQ>kni- zSZCOOuWbpho_d}Fc{l7cJ1CQx3Oph;hC?U*NWjZ9SXH2D*!L0Xj}( ztT&h-?|*&0by$>LyFM%-Ap*l7UBi%qC@tOHh$0~k12{C&9n#$$f}p_AGjvNgNDV2e zgh&mIzwzDg-uv<0-|@{q_c8Y|vDUiRy4Drvd9IN3S)e8!Ft{%SCL`m^z+4PGlnZNA z5j*T;UGV2>QFCBrGgvMOnEvp7Dy-rSl#kxC9Efs_limK;Tag#yY*s*B4i|{Ia_?@94k$Bu3+v5 z*b`C_Qm2Fawuf#3j>A7YhN#o4s;6Rzgfk`W+;0xExdQa{FRT$9doR3chv4Vxme?}u z+3WL)Hwo7R{dx7Nu$O~-*JaR=N!45|UR;aW3Sk#d$)hTmLzfqrDN7i-FeqrTS_(Vl zdj=n)%N~a4rRpEP>}m|vCIRfPk@RJ7`s~fZ+K;N~G+XAJfB2DS;n89SP55)KN<_al z^Y3jWmQ0Yf992!ehmohuh<$AJ05i!TLD|V7E{N5X4q$Oe`Q5>theXJo#Ob)uEibgQ z-SM_%RPUU;+^`v`GEHC8{KKHsqIYlW#5t%k3IMt>T*avUM&N<1ZE7M!1msEb10;6& zeH@W?!O{-8upuUSn^}Z1Vmuh>bFGh9pWW2`^u{4 zi?4u%iLc+_kXt1~&CJZuy5Uq>JAf|_8o*6mL2DmN%<9*WP51D3Wlb3JY!7+T3kV1_ z53*Y(^;SildyJ4-Vnj;a5Aww29lOvLWVCq2(i10Ik;SUum?=po z2vk8oIFvNsx9YhOYpfv_sQA)`RRf!q1XI57+jh1if=<-^;%#+|N1xwYDb~qR5jCB4 zFEWEg`$gZ!$ezui;mFPbl87CgO^elbzq_1*M}4MYe*g^+NIK<|I}Y2>h*X326Bpyi*+EElU!NrB|>svefL^*?zR<3 zVY@&vO!+nR)BdODD1UWi1_m6iajSQ%X8SgeS$)7rL}C!RRowpj>bw~eE>U*R9!^-q z3TmC03{=Y&93_AH<~u9LpWU=F2VWlI_&8UH203yLz{XW)!%&yLL^&Z2zt$anMcHYb3ximmb$>7(4a4T=|xjL~q`!5p-o z=aN42%*e`K(y!>CZ40`Uy_uDYT&W@<1(fYb4-?S2KEhM-@oAy+SaNosGxAs<4*291 zM&|kByu53i(R+{0%1KbVjRpQXs{Gga?CBDIyl+kifXC2;p1U}vQ zPc-{m!wwjf1qLuI{UWpQzG`xXPt~-?@uGg=F2GfTBV^i%p$~k>)9`2+v5N^4B42hW zDV;xGv7h#FOkQh8?q+Sail3X)<#UXN)3!}|Ka2>h7AE<$k1Ig>DzsA>XA{GvOJYFr zRr7=kcGItSy}rfa7mp%w``^1iWLLLC6Rl*Zt-{=vSa=zrh@^U!GJt7Lv=W$yc$o+0gZ-uVljyc3nBHMcQm z`grl}H#z;}M>SPn(xc{@N@C@S{l4{d*YtlTyalrm8Z=h>x6d4;8p`)@K#|T>Zu^;# zQRuUJ?{igc9)Sn3MO@+GHa5~S8B9dq_P>QQApRf)+;5Kh7qbMQH$gE6z7n2_(eB&j z0+{H+kLeJF$_VU2glNHZA6$Tw2)W@!L|?vLE_veU3s`F}#E=f@ZR^FqEy;E`7UF*+ z)`92ch$hb!!M}PQ^0ZI5EIIWM{$#?rG<6}*qw}5UR9PDnJ*@vDMxk3}e85+;RkvwF zN48K^M32G7a5d^7W{$Obr~16DcJ^te@#mcr2DU;v^&oP}FW*|3!QCSCSV&cgr}@zM z=_Z1cREwN(zqWWGT=}@md1Zq*V&KhfuV@V62h#rH@O`wnmkqbl;ozxtpvC&a>th)i z2rZlXSd%XD*rME|ddk4@T69nE`e8O473@MK00vutLGR)ze6Wqzj%s%>V%pBo5P+MH zz*6}-pSx@scea?Tn#Zm+_ZGVWS zc;;@FNOjB0jM*a}SQ84tT2)lyN`A#1)@>cL(~GWuKh7_BzeAABASL&Qav{2vqLTw4 zZm3jC$D@4^>r$}%4;*UKg7*5JIV5Y3JY%??-jLM=rx#dED;Kk z+hxW7m0?y$7-ZL|oQA?Xkd`p7s$gNGR4Lc`$<$vwM~{D-(CsfD>Y?4PCZ%E{!oe)} zF(tm~PL-NCmvqi#DJA|y0yWfh>rjW=UMd{++Ms{?y5e{5++u~!vlkLZRYByIBiLVB zN;J5IK#Z-MArZd|+xTspg|6_%yy~gd_mkaP;hp8)i%6c34yHr?2*r!2 zLV>NG`{P7{YSj{GY?09AS`W^ljKm0!>n0jsp4*sE+1y@em|9aLZCX4Z$A4Rhd^ueE zfqrOHmTLTQ#It_c`%`3l41*U8Wvxt96u0wjJ`LHt$1Nw;byq@#w@GW_AVR6@<=@uj zhu%f-r}~hrDqVkuDgO~8meFu6^R{>p9RR=v_6`vGHX&S<1Y-C_z?Ed!FU%|}SInFw z1!S4()4q(!Fp+pFIX84fTc?a*O-(OlItoo%@o~eNdi&jTVreLOPqPEe$P&UF>iUtX z{k36(ue#u>DWY!d%5t*aqU&%zXVte#U`C1H1u5o-Z6e?U<72yr0Qy-w=QrvSlrPci zIZ`e#{FR+N5Qr%YNcu2qA;aKAir>ehOEI1eVgwM?*V@R=pP0+)$q6wNp6AqbO62SG z&r4B#Xo&hX3=0hlkcy+#*tp?^rw2YAjgl7fBIRvNR`BbE+} zfQBC#5m7wzOH0Mh&wKprc$~w(j^#`KPUm-;36xuj$gkLVitLx@18v@JF#K-RbR9j6 zUnMdt_KP?cJ5jm=Rqd7seD0X`hwovz4}rM^g{+%rMuB^JSIoAOo2|?2wEzR+I4`0B z>fwzjgM>A*;)A^!KT*+&hSKqp@hk^*KI@3^fz_{(ikHGK93Y7iay`+!@80`9Jk>j| zB-&e9wdb17f>|lF43Q27g(l<>exEQGI-1Bk0Wr}QVf=XIT@P9m%(8D9RPjkiR^l>L zhjBX8XR?VQaspxu02c=8m?0oR`f9r)tIQzG%eB6KtmhoCJ83!JpQJ$sOj^EL&Lfmo z43ZUIc=@Jn1an_xj?riUf9vM;wvwKBiI;qJU?a(8R9}H=vNQ< zM%c3nv&zAF4hcz0N#D=@3&X~cKhUauw9v}A2jk@+h3$#BXYt|j*UyrDOb|i5t&!b+ zY>pU#K6c9t+k%i3?mJHe8i?BZ3b&7*W=nycUU4aL<)kJ;KIq5m8)I`xl7stAdCdxj zQ%yZ-0%GPLw}o|A8+E>Hq|};Y?%3mBIb>m{;T?uh$8Qq0owvI=cgnK;GG_jx{91is8qmZ)?dxwkHcGo{r}LU9S^x-9SMntC)$X`8EU zl0xD}K2H4l(CJvtt0rD-KiWCGjPqD7d&8F*t(P6?ZDI1{+0c_d`Lb&h-5?@uy9aR{+zSU$qbdE_;EaHR$s>Wtxq{@3LSYep>AH z%)npdkZb_q2tOlmKO`psVK87pD^J@E_GhY~2%aUxNZLa$vu|L)I9K1JjkubDJlDKe zX3pJpvGs34d`(&Pm|1d2$1^I4dUuRte(&-tuG6Z639w~Xe$d!6{knsy=t-jw!>pF z;LYc|ikZ6NSIDWB7Q3@1NWeuL+u}Mrt8h|)v6nve+At<4@H-;<&FR9y>VoO-tLE&A z@~V}RaogIsT))}k_}^6l=b7%v%YM#WenuM!5J=1;WgH)un)rF|l$VYCM8s)CsA&`n z*FYx5-r9~?tAJ1^?21H(Ag=Bk`x)mxA14huQYg4G zd*vswa%O$0NQCiI;Mtz1!cnk7#3rX;7LuE6>wfxp8IwmotIjBoF6DDPJ88M~Mxng2 zMY>?dqYJ-ko<1iPMJE~45$;Qpj3?um5%;%0WSrKRt%qyUSf)8YU^J7JrLc%U>Z>RQ zn8r~x;2p`o2)@+Kcyz2*TwzXG_In1Mj)QusEr0FBCaS3Fsmp9Xq7gZ4J=w3H);L)O zI&T0!breoEi@NLeSWE0B%gr>^U4k?9m1EKtk`+}t`{R3Xz7qYyl369)VS9Qq7nxya z+_BvLtLqH09;|B!(x(r0y*d7Yil#YVcwV_+^Gu(oXP0O1m=^C!_Ec?h4CCuz99PT1 zZoEB!=Tn0~xj`?QsowTM-}7vq;`fP!q1V)tYzdQXBQTBeqtK9%%3H04b@a{`L2>zu zs^;Cw(oH--o z#85M~{Vp1r2}C1AKa}oZ< z)Y;)q2Ygsiw#g5noc>7{(ASp6@k5jBq$VtHlR<|LuhKJfcQ^AkxWKux^9yV7E6SSP zQ`VNV7UFq64Vj1xxPaNUiUNYe}s>mXF;12!(&W&)Yw5VsONi>c-p8fRR=i_prq0#ve zu^hJLlTBH8TH zSV(-BJ=sTKeL__Bgn0EBoCAOSYZcnPNb9rAonBJh4> z$%JHG}g#==MStPdEr+>@T5Pv@#y9thB5ag+un zyY?ZK^5GaGTH?>|(==q1FMHOYmOSx(bE%=ZD@1Vqr;oRN|JYMQ)3S&6A0x><`%Nc= zN%lbxU$EG%u2{j=Dfmie^bJ*;S`};!COqoj{%AW8eYcA1EpNkZq(lE4UX~(Dwc?M8 z+Q^9Q=DdC(TP~zoHB%m!ckR#Ya?&z8Urf5Z=MtnM(ap_Boj`&>!-4+Af+<8JB{)$p z#|Zdk!2rBR{R&@VtxWcg`=-)HMBxDTxLi+-_nqScBUSUDhZVxIfu5mE=Lq}7oRsSD z6oBU4M9Rx#U`bxs7xZD~TuB1MTNKLpNFYB;MNo*zBiktZV*jCdK^L@HYV3JG5P+8J zX%wc~w)yI{2Wvnov*lVs=PFZMeLHt{owf+Co9k(Ac@=jTeY0(kdGObH$K`>F3F%sn z4&E%NwO~;>Ws0jtC3aFaNjYfO=OLR31^o(ryvx8xYZ6mmUmjmHOK+^?b`1t=!;#0H14qOm2UTMhZI&r*XN$lmMyb zMnw`hsxynLGiH8bC(oRg_5#7&0k|R5HYELbB^l* zpm-O{%+Da-yS@bzDS~}^iPyZl{aOX!1DvK<-Mh!T?ho1$``G-H)65;QVZ66kH=~9f zN{vfXRFoLX3}TYzoJNa;{8_I5#j=Vdk2&2@!p5Cne zzHm^nlt@%DwmF70zPXgd)VFl$PB{X;$G*=e`lQr^!akV_Dzx{&Yp%RA%ya!v3l33k z==!)1UHw7!>P?9_TKVR@8G2jln6Fmpemk#{w1KzKDPct_cb+O>yvrfL)0__;he8%R8f5oDH(qez>AbRaFH^0;i96xtY zJt}Mr!Ooe3)vfC0h>dDAIz=8_EQBCl678^zz=~1xmun7&m!TVW3-|`rFQ6fww}6LA z53zd|E4!l&0CT?VvDp@%jc#{KfgdiV-E@+sO0BPNA?2BI9;b}>1K(?gbN+Xj&}le) z?6Ujl7HI%yYObM5oL`hj&2Xg_P={Er30I2yHGn}+3kPjeb$ZE5Y zWwhR(v0?&HNi#9!OWDsE`xu0mEJuJ1=h^$mQHBML5~M29jiS zm}ZGz6KlZBSFiZ?knQ~jCD{l5N_AHfUZ0JAVP2lRQ&|Kp`Mq-864NUPODY+%&zm$I z_wH4>{%UgAaP7EYHN9n08xa1qEYJE5ll)|4%qdcLGBaWhji9?mr&+A^t~(?qe_nJs>@6)?$0w$(=MX>^V-#%l zMEGI6)X5Y6LYdkmz4sEe0x?sHmGzrHEgPfa_zp@O-*#gNH;uoUH{W?9)K5OHE7~v| z&||&*q6kD$KPo_LsVn@5%xB%N4RFANK|1*%|0_0Qh{f6A95}|}A)1JWXA05$@7fN! z#It52U7)1!K@;ypTCI`jaUg&7Hy`OWkz#CyqzrOObIiwGbDLVwR4F|{$CZ5~5WV+N z4)mjMJj&hV={5IDUUX;~#Rrz?&6xzWjYpB$SM~ZAl-4~-9*$nJa#KqJ3@s}LxA_VK zmL{1K?scyr6NZGlJA4XMj}@stXd#>XHQtDfYWOb5UZ&@TY zAf5RZa5OAk9PaZ1S<_Fh?ufAdjiiO!=i!Av_^#C=Ae&|wEc|1FA?nzIu8}p=mF`-h zMY!^dbgjA0Ef#QVWtB zi9##OYYut^?^W{B2EVaZ4hScilH1U^wE zo)S5msTeEuxWm{!2*^)p2<7dzaOx|CI5BJyKDZ)7t-sf)e7)}Gbu4aFyS4i=m4n!& zD-kLIh8-t|A2U)h7Y2PHMAQH^Q_ho5wDDGX&i#H99@zjodzs6@-jNXm*jUW3BXEiA_KrkgUp6BZ80{Glz{yO zZi#_3FNjuN=gCOlz-zmr=cb_nAS!m846CR%C?-1=#&lcjBOG;r%TF~GkP@GJ0VsXPjmyTc#h;h9mnvoz8^Q58fvLk$UkKN_xy0p zYWANi2pE*e2(Sd9HlHRXMJO%?F^MdqZ3Fh>u26#7AW|`iff`{NSdLsz@wQ7UcHp&w&AM4UfV3nMiyqlIf5c` ze2)iz`tU*y!l|d0Dtm94dCrHa{G_6ZoB0G~#6x8WfXG0#q&H4;S2Qgx_N9=#EO8U) zhY>Q2w~4wqM2zb^N=o%{5v17oXKwPx7`RUK_Y8v`b%GyhGvh@a2|X2_{Pj@eYu;(1 zqR;bmaKQVBl)V=V^@d)mMz;tkH2q+jzmtGmvu=A9vNc)XGUEFx(?6Y;TYPU0ha@Nv ztwRC;pcC8ZXOYrrh4}kf9Le9C(q?9nR`?_^(EGMHB<_tXjZJC!9y3h#{7z}eu7Lu8`}Q|-3VjAi!Gpa_*Z zyg*XmNUt0wh(d#z1NLsLi$Mv`S;Ua@5&zvs(^@7iW~SZ^Fi)h!qbkKECDha3oU+sPnX}yI5BC;Z z$F8j5N}$83Fa;@oX=JcS{ryik6_W_}6tz7(B+TQmp26R&6dv93L?4PUpTUG>PC@#) z9O}JJXzcnbaG>_-hbqvJ8CezTSBBL%ReE;aRM!MNW*nW0ygz!{4dV_rle$HJc zO9fWV0?&j;Mdpf>Y|^NJ!NLX@_I_Z{8FH6-jM!%8yo5;f?lk!TzvE0K)8e2ee0J5& z@UUsL{dU<9<`Ghl3tZdDGe=1A-UQ1>3y|tXkD3JP>$jBvDwC;Fc1fuwzZkldQFJ9@ zwLD|ivnv?D+w6aSM9=@IO}v8Fq;|}Yg(M0Crnbs9H3=n_Tryik?8BGOIb$Of_V6LD zmU##ine}3D@>oqJ9Wj!N;Wxz$AqzW&u3%SENA%Epn^XZ1*X&%==q^NZ(g0HD3HBY# zOk#*X)dVIag42GWW2ArdI}RT^1HVk;q3P97zh3_taG;m(>Y7GQMrD@mTh9G{nuvF4 z6OPl!tct>~P|n#L$Tah(bFoXRS-2*)>0St8qbnA15aeTj-Dfg?o9$4)7|`D9P0!K8 zZftJfA6YPxlsfy))+kF}E{nhCwN8t`Ks|wT2eV9QFZo`Al<(CwD%$hZplX(3e>N*@ zxnw?!&J6Gk8bULh-$twB&r7QNEp28gcJCMkDS2c+}_t@Z`rNV{Ey- zfTMo#|G;Z*7_?K_R`P2DV@%2M&n`-ZMBuUN0awy#pN}^`hc(#K93rgvwhf9(7LBUE z3mLlHsMMjgPkoogb7z}88V;6gW>Dxu!F=jh>kdfv=8q!WelD09wJ?y|E^GL`T{72mmGNy^ZzOVKEtb}#D`fC~_ zWCW4+;{eQOF7P7z6!p$UalZzT?+p|Mq+FC(|9;!jhzAD4`K&iOc*o3M9`D)c5 z(c9Qrwhw$eUt9$it0@Us&G;cHXcvV&XxQ$bE(#hN@*L9)!teL(0rZV~=9%3U12jiM zwc96N)arReXk+Fh0)n?Zy}eV+Bl>m-ujNt1=GXqmx%229(-a>yn$=aVuS3uh-qW|Vh{6mxN6+6u1)m*-x5q6bj!F)Z)LdGU zZLge{r*b2+bvI$deiG`{w2?P|NJRha=~o~|`>|?* zqs?3Rk5Y*l_mv|_Fxe<@=(?h?VdW}&!e5EA&`s)!Wqv$w#R59^slhPd7mLpbOS%eY zjTCdiB86e-2&hq(`ASYLizAw?){#8ZW_tJ6$oJmv7&{xv8x!eC&v*B%G-&-u4z+H9%M#CN#@!`xe>|e%RiN2>8v0y6Jes1XE~eh|Fy3J}l~> zFfT@+;D+NXu-W?c`-19nF@P_2)jUjMXk{`yjN#3Hb_X3?p|um>0Y86e*3iujy--F5 z;@Hwt7t`5%C)Mx7Cg?8E|7@U=i^`aOjg0SbiHCK3D|bh-bU2LKuvy-{Eam@B=qoL& zZzzq$(k2844jLi!MuV)b_lEmSsEV8TP z-#vL-?e=rUq2ppq0I_dZCs6*fq@G!=g>#sbBmq5+Sl}I0`-)HL`$eh~jdwCva* zr}x2Eg6%~klN&{!i!mDeuE#qI-7`@8yQD2a*GgR*$9%@R&n3KrfL_zb#IuX+;}Y`1 z>v+K9Tz@H4mMpA&+;6Qe~gZN3Wd55AU@lQW1nzVz8&hEKao948!j ze#cmJUo}GPRUXdAi8|I=n;@39$G#(s)n`Nt!Ww&;_!<0=)%1G@_OI^jAqGB9D^!}W zudi*h^J9Oa?ds&Ww5ET9_x^oB+f_Lr|H;d?){dS0fK2<=qg&uU&8F+5W1EXi|4kH3 z!_Z?cP(7Q^bVs~qfqp@(ABlS7h(0X3%|5|L{x(Nma4(d=u4{MN{_gmdp!GLrg91Rm z^T*_5PgjfRs?(qTw0fq6_oZ=P+&wD}2~1J18v242`~F7k3%Pf*$a0hK;m_rt^X<^R z9@M~XmX~U%#ZFkh;dz>&TFw`wJZo0*fL<*mFBh(B<-QJGzxy$jBPCXUHlS}L!Pw^K z3bVdu&AvSCs6ok>-#E08Tg9BWY3zSi%kR_E`YV+EvysH9U~^E1NsQ9E?fkwL-)q{F zgt>hKLv0uBxeR2Bnqbl;x@y_waIyNC@eGIjYlCocC&v!1H~gal1merY3wLcU%GS+q zxO9aI28(xlaSSMO><^A%84@0e>iHk_%aPV)YB2E+3y7>uySV8lS7LbAXvOCEB$rE| ztk-MB@PpO5MbrTjLbr&RZ@15eS@#peOjvcEOsqWlP)Oxrk9-Kj;Pq=qvTC~gP!qIY zvZoM-rsW1GB)c6Zgf!JNw6%d>@;2@M*lqZEdkf|}o1LCqr%oXKz3-k4zj?`K4hMlg zV+eBuuq=H3=e~)CGBJWg*=g%(vXORO@*Ng50eAVct4t@wVMe;E?1V~K)Je6g3JoKl zA@!&8@;l?z@ym4!sDp!QssnX{Rf`fx=ae^%YL+}RKfx;oJ|l5(ev)IpG88T{s5XsJkVcM zuSs>jkQOdtzxDWJ-?_A5s6lr6i0JEQuH)ZYypT4RgYa33ABUo2-VF}#4sbU^M4-|V zHvT|>*1*5Ry#KkN(UQOJX9iqdBF>Jcf|rgl$_-mbcG0^TMq)^V65btktoqq(O=&;* zm}BjZm}8{TZKnF{(vm6Hs5!;0kqT4ohk(&6!G5JRK4JQ&LnpWPUEv;A z#8)8U@r=;jlw`e0#dTolsm2cq>fo@l zVc4W~Vq4%G*?zF>;Qh*7h{0+7+h&m}GJfU#EZi-ft7*MQ&~=d7(F9h{z~3Oq-zSzNkZiq^}^S#8{VZ#vnRiy20>;Y=8!z(zq-n0`H?Q7FZa(@1uM1QyIi1zqv zZ`lt%ScgUU*>Kfn=v2yj@0;g z&Y;jS7Ltc#uKOrIyZD$ZykyfS^|8B0)L!-1Q@z*p2kx-Ij8br$yt+*LnIL=)$%p3i7$;%nZ zuBh>g6A(PC@0*3ds8wt<)pV?ba49r8&PmWDY=2}(=6p^% zZOj~L>v8R@aHwxbmwi|#-*H55c)359LZ`LV)rPu?Vg|i>&aL@(T!5C$V{!@%wQD=@ zZn@6SFM*3~M}^uXcC#iXr0FKci;w^l6Igp|%$sBK(FhU_aX&_U;#gM>1U9!693^kGKjVM4dF==4_}#Lr!gbUAdIl3Hx{X(vhU}iw>$t z+zOv-`n}!#SGK$@NhT^L@%;2)##?$#a$D>IYckEBIT($Bk;wc4vI(*3N3;w(@b%KYhUeQJ0Pc^#7nETE%h8`J1ZKt zY|WOt@^n_%1(@wZg%Qd+&~o)`@lo<84s{#wi?yly>VggOL>sy25T?9Gy++U3vYqRh6n@LxOaLtLox#8EWOzJBBqpf2Nvqjh1QNaaEXpFGOvInCoD@P+GPs@AVMz)ZwpWfXhjGdjSauaP<-*bPc)WgWfp%l_6pRD@}>QGjPK;WP&veta{@L8xv z`3l+@`(9h=XXF5RRi;J}(^tfR`C*~)MvpqpALcSvVPqsKXt3j7XZ7qhfhzz>w%brb zNpC7nT5^vyhWPSVQtv#+fS3AzBs~83Ghwk(L7{6IL_rBo9d|kb);^9Gf$(Qp_M-;} zQ{G}=`v`tUbCR>E;5E{9f5r^`R}o5WIqlgKDN4|(i6uSdVR=Ck%tSJ-HO7PY5pq?K zN?_*qR87TvB^(KQmXFdF!Lt@pKK%5zPa?*QnLl!0X}Hxwn=eJ6 zVdI-GA2c^D()k%b5P+$zFu}=BXurZ|BBeS7@PJci;YG(~-}gC5H28-QdI`YW*9JFY zLz^KMN)M3n$1gjv{0_jNX&AJ$o}LcknuoG>?>?!lPZRD>htnryf6CzK*S`Mtu@vWv z_L$_~^Ea}S62P7SxM6<-Wz;FdJ**~t{abR;wdY!m)Ixu;l_iT&6-SfMr~WJ|l%Ey^ zs@!M4jwm>CGVaVT163tIIATvGVxtGsRZs~PWL9#wm`hGny}y5L0E5o4?kzErR+5g_ zkv+TMH1tGGzXUUXR&14ZHwtkRYpyISgSuoqEQNzV8UzR|D=C&EAo<1(&~w>Y_-lWr^0B&J%>Wpe|Nb=0_R4o_@0sWr)F=UK zC@zwb;jMR;DTd5{rXC91@m@wmD9LyQNx!yN`gvGS*KtuJesMd#n3ug`k5Jr{n*NCx z;pZ}Ld?UeNBUzmKDS0DIP`!W?ETdmr<$@zau(hZC!YhlY+>#J1JY|`ZReUx94q5hr zm0E;-n(k9XCGvZ!?X35rdw;y$4%)jO^Xag}5@ir?9EVv+Kzb=XHsWs!Um3a7DnI{^TE^kmUXYi`*yQ?9UkR) zYMrk=YWbp(GaG^gGh^U0yX608v`!oJ0KafaIyM43U?d-pcO2PaUw5$3y5jEGIEs?2 z9&1Ka)U_K))+d#$3#2d#W#~Ng4T`N81#|Lg)@2 ze@7W|rg+#7EJMpPY`YZ1L{PGpuEyHNqF<_rL{eM@FO?K)aFwN^Y&QS!ME-Z?chB_5I~b! zfC-UZ!cgVE+%0t86oWAVn>k$i|9nS{5-q;T`f#^`{Xe`QCG4cYP0D{{T>srud|43w zK&W&s8Jha=Kd^-VY6&ob7VAInfw|4_&_l)jsqVvn3>DgXVAyaa!Hw>k8s%4g?f)|x z%ou|_{~x0oh8|V?5E0J5o!CEP!weF}`9FVa+6_>`mjAo?f8Jp;_y5PJp&Rl4 c^APR=@*HC}1GooB@1bAH@)~lLGN!@*55u8W`_, +which relies on absolute evaluations, preferential optimization significantly reduces fluctuations in the evaluators' criteria, +ensuring more consistent results. + +In this tutorial, we will interactively optimize RGB values between 0 and 255 to generate a color that resembles the "sunset hue", which is the same problem setting as `this tutorial `. +Hence, familiarizing yourself with the tutorial on objective form widgets beforehand might offer a smoother understanding. + +How to Run Preferential Optimization +------------------------------------ + +In preferential optimization, we run two programs simultaneously: `generator.py`_ which executes parameter sampling or image generation, +and the Optuna Dashboard which provides a user interface for human evaluation. + +.. figure:: ./images/preferential-optimization/system-architecture.png + :alt: System Architecture + :align: center + :width: 800px + +To start, ensure you have the necessary packages installed. You can do this by running the following command in your terminal: + +.. code-block:: console + + $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.13.0b1" pillow botorch + +Run a Python script below which you copied from `generator.py`_. + +.. code-block:: console + + $ python generator.py + +Then run a following command to launch Optuna Dashboard in a separate process. + +.. code-block:: console + + $ optuna-dashboard sqlite:///example.db --artifact-dir ./artifact + +In the command, the storage is set to ``sqlite:///example.db`` to persist Optuna's trial history. +To store the artifacts (output images), ``--artifact-dir ./artifact`` is specified. + +.. code-block:: console + + Listening on http://127.0.0.1:8080/ + Hit Ctrl-C to quit. + +When you run the command, you will see a message like the one above. +Please open `http://127.0.0.1:8080/dashboard/ `_ in your browser, then you can see the Optuna Dashboard as follows: + +.. figure:: ./images/preferential-optimization/anim.gif + :alt: GIF animation for preferential optimization + :align: center + :width: 800px + + Selecting the least sunset-like color from four trials to report human preferences. + + +Script Explanation +------------------ + +Here, we specify the SQLite database URL and setup the artifact store, a filesystem to store images generated during the trial. + +.. code-block:: python + :linenos: + + STORAGE_URL = "sqlite:///example.db" + artifact_path = os.path.join(os.path.dirname(__file__), "artifact") + artifact_store = FileSystemArtifactStore(base_path=artifact_path) + os.makedirs(artifact_path, exist_ok=True) + +Within the ``main()`` function, we initialize the study with necessary parameters, including specifying the preferential sampler. +ote that the ``Study`` and ``Sampler`` instantiated here are different from the conventional Optuna's ``Study``` and the ``Sampler``. +Preferential optimization relies solely on the comparison results between trials, and there are no absolute evaluation values for each trial. +Therefore, it is necessary to create dedicated ``Study`` and ``Sampler`` objects. + +.. code-block:: python + :linenos: + + from optuna_dashboard.preferential import create_study + from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler + + study = create_study( + n_generate=5, + study_name="Preferential Optimization", + storage=STORAGE_URL, + sampler=PreferentialGPSampler(), + load_if_exists=True, + ) + +Then, we create a loop that continuously checks if new trials should be generated, awaiting human evaluation if not. +Within the while loop, new trials are generated if the condition :meth:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns ``True``. +For each trial, RGB values are sampled, and an image is generated with these values. +The image is saved temporarily, uploaded to artifact store, and then saved a Markdown note using :func:`~optuna_dashboard.save_note`. + +.. code-block:: python + :linenos: + + while True: + # If study.should_generate() returns False, the generator waits for human evaluation. + if not study.should_generate(): + time.sleep(0.1) # Avoid busy-loop + continue + + trial = study.ask() + # Ask new parameters + r = trial.suggest_int("r", 0, 255) + g = trial.suggest_int("g", 0, 255) + b = trial.suggest_int("b", 0, 255) + + # Generate an image + image_path = os.path.join(tmpdir, f"sample-{trial.number}.png") + image = Image.new("RGB", (320, 240), color=(r, g, b)) + image.save(image_path) + + # Upload to Artifact store + artifact_id = upload_artifact(trial, image_path, artifact_store) + trial.set_user_attr("artifact_id", artifact_id) + print("RGB:", (r, g, b)) + + # Save a Markdown note + note = textwrap.dedent( + f"""\ + ![generated-image]({get_artifact_path(trial, artifact_id)}) + + (R, G, B) = ({r}, {g}, {b}) + """ + ) + +.. _generator.py: https://github.com/optuna/optuna-dashboard/blob/main/examples/preferential-optimization/generator.py diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index d8f7d3bd..66be1765 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -6,10 +6,10 @@ import textwrap import time from typing import NoReturn +from optuna.artifacts import FileSystemArtifactStore +from optuna.artifacts import upload_artifact from optuna_dashboard import save_note from optuna_dashboard.artifact import get_artifact_path -from optuna_dashboard.artifact import upload_artifact -from optuna_dashboard.artifact.file_system import FileSystemBackend from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler from PIL import Image @@ -17,7 +17,7 @@ from PIL import Image STORAGE_URL = "sqlite:///example.db" artifact_path = os.path.join(os.path.dirname(__file__), "artifact") -artifact_backend = FileSystemBackend(base_path=artifact_path) +artifact_store = FileSystemArtifactStore(base_path=artifact_path) os.makedirs(artifact_path, exist_ok=True) @@ -32,7 +32,7 @@ def main() -> NoReturn: with tempfile.TemporaryDirectory() as tmpdir: while True: - # If n_comparison "best" trials (that are not reported bad) exists, + # If study.should_generate() returns False, # the generator waits for human evaluation. if not study.should_generate(): time.sleep(0.1) # Avoid busy-loop @@ -50,7 +50,7 @@ def main() -> NoReturn: image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) trial.set_user_attr("artifact_id", artifact_id) print("RGB:", (r, g, b)) From 2d7e2f6bc51f2cf4a1fb01af9478459d66861cdf Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 8 Sep 2023 22:43:59 +0900 Subject: [PATCH 037/104] Update preferential optimization tutorial --- docs/tutorials/preferential-optimization.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/preferential-optimization.rst b/docs/tutorials/preferential-optimization.rst index 99f68cf4..706eabe8 100644 --- a/docs/tutorials/preferential-optimization.rst +++ b/docs/tutorials/preferential-optimization.rst @@ -10,7 +10,7 @@ Compared to the `human-in-the-loop optimization utilizing objective form widgets which relies on absolute evaluations, preferential optimization significantly reduces fluctuations in the evaluators' criteria, ensuring more consistent results. -In this tutorial, we will interactively optimize RGB values between 0 and 255 to generate a color that resembles the "sunset hue", which is the same problem setting as `this tutorial `. +In this tutorial, we will interactively optimize RGB values between 0 and 255 to generate a color that resembles the "sunset hue", which is the same problem setting as `this tutorial `_. Hence, familiarizing yourself with the tutorial on objective form widgets beforehand might offer a smoother understanding. How to Run Preferential Optimization From 055a9cb827b3860ab5c23fced50c156302576158 Mon Sep 17 00:00:00 2001 From: c-bata Date: Sat, 9 Sep 2023 01:08:05 +0900 Subject: [PATCH 038/104] Update docs/tutorials/hitl.rst --- docs/tutorials/hitl.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 932c03a1..42e163df 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -1,4 +1,4 @@ -.. _tutorial-hitl: +.. _tutorial-hitl-objective-form-widgets: Tutorial: Human-in-the-loop Optimization using Objective Form Widgets ===================================================================== From 260b76094942ce58f4bfb4af7331847ac7496501 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 10 Sep 2023 15:50:37 +0900 Subject: [PATCH 039/104] Update hitl tutorial for artifacts --- docs/tutorials/hitl.rst | 27 ++++++++++++++------------- examples/hitl/main.py | 18 ++++++++++-------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 6b0cf7c7..1df12973 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -97,7 +97,7 @@ To run `the script =3.2.0" "optuna-dashboard>=0.10.0" pillow + $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.12.0" pillow You will use SQLite for the storage backend in this tutorial. Ensure that the following library is installed: @@ -179,7 +179,9 @@ Let’s walk through the script we used for the optimization. .. code-block:: python :linenos: - def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystemBackend) -> None: + def suggest_and_generate_image( + study: optuna.Study, artifact_store: FileSystemArtifactStore + ) -> None: # 1. Ask new parameters trial = study.ask() r = trial.suggest_int("r", 0, 255) @@ -192,7 +194,7 @@ Let’s walk through the script we used for the optimization. image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) artifact_path = get_artifact_path(trial, artifact_id) # 4. Save Note @@ -210,7 +212,7 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new .. code-block:: python :linenos: - def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: + def start_optimization(artifact_store: FileSystemArtifactStore) -> NoReturn: # 1. Create Study study = optuna.create_study( study_name="Human-in-the-loop Optimization", @@ -218,10 +220,10 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new sampler=optuna.samplers.TPESampler(constant_liar=True, n_startup_trials=5), load_if_exists=True, ) - + # 2. Set an objective name study.set_metric_names(["Looks like sunset color?"]) - + # 3. Register ChoiceWidget register_objective_form_widgets( study, @@ -234,15 +236,14 @@ In the ``suggest_and_generate_image`` function, a new Trial is obtained and new ], ) - # 4. Start Optimization + # 4. Start Human-in-the-loop Optimization n_batch = 4 while True: running_trials = study.get_trials(deepcopy=False, states=(TrialState.RUNNING,)) if len(running_trials) >= n_batch: time.sleep(1) # Avoid busy-loop continue - suggest_and_generate_image(study, artifact_backend) - + suggest_and_generate_image(study, artifact_store) The function ``start_optimization`` defines our loop for HITL optimization to generate an image resembling a sunset color. @@ -256,10 +257,10 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge def main() -> NoReturn: tmp_path = os.path.join(os.path.dirname(__file__), "tmp") - + # 1. Create Artifact Store artifact_path = os.path.join(os.path.dirname(__file__), "artifact") - artifact_backend = FileSystemBackend(base_path=artifact_path) + artifact_store = FileSystemArtifactStore(artifact_path) if not os.path.exists(artifact_path): os.mkdir(artifact_path) @@ -268,11 +269,11 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge os.mkdir(tmp_path) # 2. Run optimize loop - start_optimization(artifact_backend) + start_optimization(artifact_store) In the ``main`` function, at first, the locations of the Artifact Store is set. -* At #1, the :class:`~optuna_dashboard.FileSystemBackend` is created, which is one of the Artifact Storage options used in the Optuna Dashboard. Artifact Storage is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. +* At #1, the `FileSystemArtifactStore ` is created, which is one of the Artifact Store options used in the Optuna. Artifact Store is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. * At #2, `start_optimization()` function, which is described above, is called. After that, two folders are created, artifact and tmp, and then ``start_optimization`` function is called to start the HITL optimization using Optuna. diff --git a/examples/hitl/main.py b/examples/hitl/main.py index 1557ae21..7844bc7a 100644 --- a/examples/hitl/main.py +++ b/examples/hitl/main.py @@ -4,17 +4,19 @@ import time from typing import NoReturn import optuna +from optuna.artifacts import FileSystemArtifactStore +from optuna.artifacts import upload_artifact from optuna.trial import TrialState from optuna_dashboard import ChoiceWidget from optuna_dashboard import register_objective_form_widgets from optuna_dashboard import save_note from optuna_dashboard.artifact import get_artifact_path -from optuna_dashboard.artifact import upload_artifact -from optuna_dashboard.artifact.file_system import FileSystemBackend from PIL import Image -def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystemBackend) -> None: +def suggest_and_generate_image( + study: optuna.Study, artifact_store: FileSystemArtifactStore +) -> None: # 1. Ask new parameters trial = study.ask() r = trial.suggest_int("r", 0, 255) @@ -27,7 +29,7 @@ def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystem image.save(image_path) # 3. Upload Artifact - artifact_id = upload_artifact(artifact_backend, trial, image_path) + artifact_id = upload_artifact(trial, image_path, artifact_store) artifact_path = get_artifact_path(trial, artifact_id) # 4. Save Note @@ -41,7 +43,7 @@ def suggest_and_generate_image(study: optuna.Study, artifact_backend: FileSystem save_note(trial, note) -def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: +def start_optimization(artifact_store: FileSystemArtifactStore) -> NoReturn: # 1. Create Study study = optuna.create_study( study_name="Human-in-the-loop Optimization", @@ -72,7 +74,7 @@ def start_optimization(artifact_backend: FileSystemBackend) -> NoReturn: if len(running_trials) >= n_batch: time.sleep(1) # Avoid busy-loop continue - suggest_and_generate_image(study, artifact_backend) + suggest_and_generate_image(study, artifact_store) def main() -> NoReturn: @@ -80,7 +82,7 @@ def main() -> NoReturn: # 1. Create Artifact Store artifact_path = os.path.join(os.path.dirname(__file__), "artifact") - artifact_backend = FileSystemBackend(base_path=artifact_path) + artifact_store = FileSystemArtifactStore(artifact_path) if not os.path.exists(artifact_path): os.mkdir(artifact_path) @@ -89,7 +91,7 @@ def main() -> NoReturn: os.mkdir(tmp_path) # 2. Run optimize loop - start_optimization(artifact_backend) + start_optimization(artifact_store) if __name__ == "__main__": From 633b6aaec8228f4145cd321aa39d7ba909226dc6 Mon Sep 17 00:00:00 2001 From: keisuke-umezawa Date: Sun, 10 Sep 2023 16:10:18 +0900 Subject: [PATCH 040/104] Fix wording --- docs/tutorials/hitl.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/tutorials/hitl.rst b/docs/tutorials/hitl.rst index 1df12973..07d2f499 100644 --- a/docs/tutorials/hitl.rst +++ b/docs/tutorials/hitl.rst @@ -207,7 +207,7 @@ Let’s walk through the script we used for the optimization. ) save_note(trial, note) -In the ``suggest_and_generate_image`` function, a new Trial is obtained and new hyperparameters are suggested for that Trial. Based on those hyperparameters, an RGB image is generated as an artifact. The generated image is then uploaded to the Artifact Storage of the Optuna Dashboard, and the image is also displayed in the Dashboard's Note. For more information on how to use the Note feature, please refer to the API Reference of :func:`~optuna_dashboard.save_note`. +In the ``suggest_and_generate_image`` function, a new Trial is obtained and new hyperparameters are suggested for that Trial. Based on those hyperparameters, an RGB image is generated as an artifact. The generated image is then uploaded to the Artifact Store of the Optuna, and the image is also displayed in the Dashboard's Note. For more information on how to use the Note feature, please refer to the API Reference of :func:`~optuna_dashboard.save_note`. .. code-block:: python :linenos: @@ -273,7 +273,7 @@ The function ``start_optimization`` defines our loop for HITL optimization to ge In the ``main`` function, at first, the locations of the Artifact Store is set. -* At #1, the `FileSystemArtifactStore ` is created, which is one of the Artifact Store options used in the Optuna. Artifact Store is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. +* At #1, the `FileSystemArtifactStore `_ is created, which is one of the Artifact Store options used in the Optuna. Artifact Store is used to store artifacts (data, files, etc.) generated during Optuna trials. For more information, please refer to the API Reference. * At #2, `start_optimization()` function, which is described above, is called. After that, two folders are created, artifact and tmp, and then ``start_optimization`` function is called to start the HITL optimization using Optuna. From b1c05cf11f343267aa0d060cb8083a6f24b703a9 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 10:57:46 +0900 Subject: [PATCH 041/104] fix by review --- optuna_dashboard/ts/components/AppDrawer.tsx | 6 +++--- optuna_dashboard/ts/components/PreferentialGraph.tsx | 6 ++---- optuna_dashboard/ts/types/index.d.ts | 2 -- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 97157dd9..e6422eb0 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -208,7 +208,7 @@ export const AppDrawer: FC<{ {isPreferential ? : } @@ -229,7 +229,7 @@ export const AppDrawer: FC<{ @@ -260,7 +260,7 @@ export const AppDrawer: FC<{ diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index d67c1961..8a09343f 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -175,7 +175,6 @@ export const PreferentialGraph: FC<{ (changes) => setNodes((nds) => applyNodeChanges(changes, nds)), [setNodes] ) - const isDarkMode = theme.palette.mode === "dark" useEffect(() => { if (studyDetail === null) return @@ -201,7 +200,6 @@ export const PreferentialGraph: FC<{ id: `e${source}-${target}`, sources: [`${source}`], targets: [`${target}`], - style: { stroke: isDarkMode ? "#fff" : "#000" }, })), } elk @@ -244,11 +242,11 @@ export const PreferentialGraph: FC<{ id: `e${p[0]}-${p[1]}`, source: `${p[0]}`, target: `${p[1]}`, - style: { stroke: isDarkMode ? "#fff" : "#000" }, + style: { stroke: theme.palette.text.primary }, } as Edge }) ?? [] ) - }, [studyDetail, isDarkMode]) + }, [studyDetail, theme.palette.text.primary]) if (studyDetail === null || !studyDetail.is_preferential) { return null diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 55fde04d..81096248 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -223,5 +223,3 @@ type PreferenceHistory = { feedback_mode: PreferenceFeedbackMode timestamp: Date } - -declare module "*.css" From 0f2c49602eb134d47062657937eceecf0e24bd9d Mon Sep 17 00:00:00 2001 From: c-bata Date: Mon, 11 Sep 2023 13:46:26 +0900 Subject: [PATCH 042/104] Add FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 00000000..c2c7a342 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +github: optuna From 92b94b64888d9b6b805b169ec2cc9e7994ee65e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 14:40:57 +0900 Subject: [PATCH 043/104] merge and modify --- .github/workflows/python-coverage.yml | 2 +- docs/api.rst | 1 + .../preferential-optimization/generator.py | 3 - optuna_dashboard/__init__.py | 3 +- optuna_dashboard/_app.py | 40 +- optuna_dashboard/_custom_plot_data.py | 134 ++++ optuna_dashboard/_preferential_history.py | 80 +++ optuna_dashboard/_serializer.py | 35 ++ optuna_dashboard/preferential/_study.py | 94 +-- .../preferential/_system_attrs.py | 27 +- optuna_dashboard/preferential/samplers/gp.py | 579 ++++++++---------- optuna_dashboard/ts/action.ts | 6 +- optuna_dashboard/ts/apiClient.ts | 39 +- optuna_dashboard/ts/components/App.tsx | 9 + optuna_dashboard/ts/components/AppDrawer.tsx | 31 +- .../ts/components/ArtifactCardMedia.tsx | 41 ++ .../ts/components/PreferenceHistory.tsx | 218 +++++++ .../ts/components/PreferentialTrials.tsx | 473 +++++++------- .../ts/components/StudyDetail.tsx | 3 + .../ts/components/StudyHistory.tsx | 11 + .../ts/components/ThreejsArtifactViewer.tsx | 54 +- .../ts/components/TrialArtifactCards.tsx | 233 +++++++ optuna_dashboard/ts/components/TrialList.tsx | 360 +---------- .../ts/components/UserDefinedPlot.tsx | 21 + optuna_dashboard/ts/types/index.d.ts | 17 + pyproject.toml | 1 + python_tests/preferential/test_study.py | 6 +- .../preferential/test_system_attrs.py | 7 +- python_tests/test_api.py | 57 +- python_tests/test_custom_plot_data.py | 86 +++ python_tests/test_preferential_history.py | 61 ++ python_tests/test_serializers.py | 4 +- 32 files changed, 1755 insertions(+), 981 deletions(-) create mode 100644 optuna_dashboard/_custom_plot_data.py create mode 100644 optuna_dashboard/_preferential_history.py create mode 100644 optuna_dashboard/ts/components/ArtifactCardMedia.tsx create mode 100644 optuna_dashboard/ts/components/PreferenceHistory.tsx create mode 100644 optuna_dashboard/ts/components/TrialArtifactCards.tsx create mode 100644 optuna_dashboard/ts/components/UserDefinedPlot.tsx create mode 100644 python_tests/test_custom_plot_data.py create mode 100644 python_tests/test_preferential_history.py diff --git a/.github/workflows/python-coverage.yml b/.github/workflows/python-coverage.yml index 4bda6350..c96f506b 100644 --- a/.github/workflows/python-coverage.yml +++ b/.github/workflows/python-coverage.yml @@ -45,4 +45,4 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.xml - fail_ci_if_error: true + fail_ci_if_error: false diff --git a/docs/api.rst b/docs/api.rst index 09a9c14b..aadd2718 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -14,6 +14,7 @@ General APIs optuna_dashboard.wsgi optuna_dashboard.set_objective_names optuna_dashboard.save_note + optuna_dashboard.save_plotly_graph_object Human-in-the-loop ----------------- diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index e94f1d05..d8f7d3bd 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -64,9 +64,6 @@ def main() -> NoReturn: ) save_note(trial, note) - # 5. Mark comparison ready - study.mark_comparison_ready(trial) - if __name__ == "__main__": main() diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 3d363cf4..5bb3f301 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -1,5 +1,6 @@ from ._app import run_server # noqa from ._app import wsgi # noqa +from ._custom_plot_data import save_plotly_graph_object # noqa from ._form_widget import ChoiceWidget # noqa from ._form_widget import dict_to_form_widget # noqa from ._form_widget import ObjectiveChoiceWidget # noqa @@ -15,4 +16,4 @@ from ._note import get_note # noqa from ._note import save_note # noqa -__version__ = "0.12.0" +__version__ = "0.13.0b1" diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 70acc13f..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -25,9 +25,12 @@ from . import _note as note from ._bottle_util import BottleViewReturn from ._bottle_util import json_api_view from ._cached_extra_study_property import get_cached_extra_study_property +from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preference_setting import _register_output_component +from ._preferential_history import NewHistory +from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -41,7 +44,6 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials -from .preferential._system_attrs import report_preferences from .preferential._system_attrs import report_skip @@ -214,6 +216,8 @@ def create_app( union_user_attrs, has_intermediate_values, ) = get_cached_extra_study_property(study_id, trials) + + plotly_graph_objects = get_plotly_graph_objects(system_attrs) return serialize_study_detail( summary, best_trials, @@ -222,6 +226,7 @@ def create_app( union, union_user_attrs, has_intermediate_values, + plotly_graph_objects, ) @app.get("/api/studies//param_importances") @@ -270,17 +275,34 @@ def create_app( @json_api_view def post_preference(study_id: int) -> dict[str, Any]: try: - best_trials = [int(d) for d in request.json.get("best_trials", [])] - worst_trials = [int(d) for d in request.json.get("worst_trials", [])] + mode = request.json.get("mode", "") + candidates = [int(d) for d in request.json.get("candidates", [])] + clicked = int(request.json.get("clicked", -1)) except ValueError: response.status = 400 - return {"reason": "best_trials and worst_trials must be an array of integers."} - if len(best_trials) == 0 or len(worst_trials) == 0: - response.status = 400 # Bad request - return {"reason": "You need to set best_trials and worst_trials"} + return { + "reason": ( + "`candidates` should be an array of integers and " + "`clicked` should be an integer." + ) + } - preferences = [(best, worst) for best in best_trials for worst in worst_trials] - report_preferences(study_id, storage, preferences) + if clicked == -1: + response.status = 400 + return {"reason": "`clicked` should be specified."} + if mode != "ChooseWorst": + response.status = 400 + return {"reason": "`mode` should be 'ChooseWorst'."} + + report_history( + study_id, + storage, + NewHistory( + mode=mode, + candidates=candidates, + clicked=clicked, + ), + ) response.status = 204 return {} diff --git a/optuna_dashboard/_custom_plot_data.py b/optuna_dashboard/_custom_plot_data.py new file mode 100644 index 00000000..a4dfa4af --- /dev/null +++ b/optuna_dashboard/_custom_plot_data.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import math +from typing import TYPE_CHECKING +import uuid + +from optuna import Study + + +if TYPE_CHECKING: + from typing import Any + + from optuna.storages import BaseStorage + import plotly.graph_objs as go + + +SYSTEM_ATTR_PLOT_DATA = "dashboard:plot_data:" +SYSTEM_ATTR_MAX_LENGTH = 2045 + + +def save_plotly_graph_object( + study: Study, figure: go.Figure, *, graph_object_id: str | None = None +) -> str: + """Save the user-defined plotly's graph object to the study. + + Example: + + .. code-block:: python + + import optuna + from optuna_dashboard import save_plotly_graph_object + + def objective(trial): + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + study.optimize(objective, n_trials=100) + + figure = optuna.visualization.plot_optimization_history(study) + save_plotly_graph_object(study, figure) + + Args: + study: + Target study object. + plot_data: + The plotly's graph object to save. + graph_object_id: + Unique identifier of the graph object. If specified, the graph object is overwritten. + This must be a valid HTML id attribute value. + + Returns: + The graph object ID. + """ + if graph_object_id is not None and not is_valid_graph_object_id(graph_object_id): + raise ValueError("graph_object_id must be a valid HTML id attribute value.") + + storage = study._storage + study_id = study._study_id + + graph_object_id = graph_object_id or str(uuid.uuid4()) + key = SYSTEM_ATTR_PLOT_DATA + graph_object_id + ":" + plot_data_json_str = figure.to_json() + save_graph_object_json(storage, study_id, key, plot_data_json_str) + return graph_object_id + + +def save_graph_object_json( + storage: BaseStorage, study_id: int, key_prefix: str, plot_data_json_str: str +) -> None: + plot_data_system_attrs = split_plot_data(plot_data_json_str, key_prefix) + for k, v in plot_data_system_attrs.items(): + storage.set_study_system_attr(study_id, k, v) + + # Clear previous graph object attributes + study_system_attrs = storage.get_study_system_attrs(study_id) + all_plot_data_system_attrs = [k for k in study_system_attrs if k.startswith(key_prefix)] + if len(all_plot_data_system_attrs) > len(plot_data_system_attrs): + for i in range(len(plot_data_system_attrs), len(all_plot_data_system_attrs)): + storage.set_study_system_attr(study_id, f"{key_prefix}{i}", "") + + +def list_graph_object_ids(system_attrs: dict[str, Any]) -> list[str]: + titles = set() + for key in system_attrs: + if not key.startswith(SYSTEM_ATTR_PLOT_DATA): + continue + + s = key.split(":", maxsplit=2) # e.g. ["dashboard", "plot_data", "Optimization History:1"] + if len(s) != 3: + continue + # Please note that title may contain ":". + title = s[2].rsplit(":", maxsplit=1)[0] + titles.add(title) + return list(titles) + + +def get_plotly_graph_objects(system_attrs: dict[str, Any]) -> dict[str, str]: + graph_objects = {} + for title in list_graph_object_ids(system_attrs): + key_prefix = SYSTEM_ATTR_PLOT_DATA + title + ":" + plot_data_attrs = {k: v for k, v in system_attrs.items() if k.startswith(key_prefix)} + graph_objects[title] = concat_plot_data(plot_data_attrs, key_prefix) + return graph_objects + + +def split_plot_data(plot_data_str: str, key_prefix: str) -> dict[str, str]: + plot_data_len = len(plot_data_str) + attrs = {} + for i in range(math.ceil(plot_data_len / SYSTEM_ATTR_MAX_LENGTH)): + start = i * SYSTEM_ATTR_MAX_LENGTH + end = min((i + 1) * SYSTEM_ATTR_MAX_LENGTH, plot_data_len) + attrs[f"{key_prefix}{i}"] = plot_data_str[start:end] + return attrs + + +def concat_plot_data(plot_data_attrs: dict[str, str], key_prefix: str) -> str: + return "".join(plot_data_attrs[f"{key_prefix}{i}"] for i in range(len(plot_data_attrs))) + + +def is_valid_graph_object_id(graph_object_id: str) -> bool: + if len(graph_object_id) == 0: + return False + + # Can only contain letters [A-Za-z], numbers [0-9], hyphens ("-"), underscores ("_"), + # colons, and periods. + if not all( + "a" <= c <= "z" or "A" <= c <= "Z" or "0" <= c <= "9" or c in ("-", "_", ":", ".") + for c in graph_object_id[1:] + ): + return False + # Unlike HTML id attribute, graph object id can begin with a letter [A-Za-z] + return True diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py new file mode 100644 index 00000000..6b81b9bb --- /dev/null +++ b/optuna_dashboard/_preferential_history.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import json +from typing import TYPE_CHECKING +import uuid + +from optuna.storages import BaseStorage + +from .preferential._system_attrs import report_preferences + + +_SYSTEM_ATTR_PREFIX_HISTORY = "preference:history" + +if TYPE_CHECKING: + from typing import Literal + from typing import TypedDict + + FeedbackMode = Literal["ChooseWorst"] + ChooseWorstHistory = TypedDict( + "ChooseWorstHistory", + { + "mode": FeedbackMode, + "id": str, + "preference_id": str, + "timestamp": str, + "candidates": list[int], + "clicked": int, + }, + ) + History = ChooseWorstHistory + + +@dataclass +class NewHistory: + mode: FeedbackMode + candidates: list[int] + clicked: int + + +def report_history( + study_id: int, + storage: BaseStorage, + input_data: NewHistory, +) -> None: + preferences = [] + # TODO(moririn): Use TypeGuard after adding other history types. + if input_data.mode == "ChooseWorst": + preferences = [ + (best, input_data.clicked) + for best in input_data.candidates + if best != input_data.clicked + ] + else: + assert False, f"Unknown data: {input_data}" + + preference_id = report_preferences( + study_id=study_id, + storage=storage, + preferences=preferences, + ) + history_id = str(uuid.uuid4()) + + if input_data.mode == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": history_id, + "preference_id": preference_id, + "timestamp": datetime.now().isoformat(), + "candidates": input_data.candidates, + "clicked": input_data.clicked, + } + + key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + storage.set_study_system_attr( + study_id=study_id, + key=key, + value=json.dumps(history), + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index b2494068..1f49e925 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -1,5 +1,6 @@ from __future__ import annotations +from datetime import datetime import json from typing import Any from typing import TYPE_CHECKING @@ -16,6 +17,7 @@ from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -24,6 +26,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import ChooseWorstHistory + from ._preferential_history import History + Attribute = TypedDict( "Attribute", { @@ -129,6 +134,7 @@ def serialize_study_detail( union: list[tuple[str, BaseDistribution]], union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, + plotly_graph_objects: dict[str, str], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -161,9 +167,38 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["preference_history"] = serialize_preference_history(system_attrs) + serialized["plotly_graph_objects"] = [ + {"id": id_, "graph_object": graph_object} + for id_, graph_object in plotly_graph_objects.items() + ] return serialized +def serialize_preference_history( + system_attrs: dict[str, Any], +) -> list[History]: + histories: list[History] = [] + for k, v in system_attrs.items(): + if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): + continue + choice: dict[str, Any] = json.loads(v) + if choice["mode"] == "ChooseWorst": + history: ChooseWorstHistory = { + "mode": "ChooseWorst", + "id": choice["id"], + "preference_id": choice["preference_id"], + "timestamp": choice["timestamp"], + "candidates": choice["candidates"], + "clicked": choice["clicked"], + } + histories.append(history) + + histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + return histories + + def serialize_frozen_trial( study_id: int, trial: FrozenTrial, study_system_attrs: dict[str, Any] ) -> dict[str, Any]: diff --git a/optuna_dashboard/preferential/_study.py b/optuna_dashboard/preferential/_study.py index ce691f42..6e093683 100644 --- a/optuna_dashboard/preferential/_study.py +++ b/optuna_dashboard/preferential/_study.py @@ -14,6 +14,7 @@ from optuna.trial import FrozenTrial from optuna.trial import TrialState from optuna_dashboard.preferential._system_attrs import get_n_generate from optuna_dashboard.preferential._system_attrs import get_preferences +from optuna_dashboard.preferential._system_attrs import get_skipped_trial_ids from optuna_dashboard.preferential._system_attrs import is_skipped_trial from optuna_dashboard.preferential._system_attrs import report_preferences from optuna_dashboard.preferential._system_attrs import set_n_generate @@ -21,7 +22,6 @@ from optuna_dashboard.preferential._system_attrs import set_n_generate _logger = logging.get_logger(__name__) _SYSTEM_ATTR_PREFERENTIAL_STUDY = "preference:is_preferential" -_SYSTEM_ATTR_COMPARISON_READY = "preference:comparison_ready" class PreferentialStudy: @@ -62,13 +62,6 @@ class PreferentialStudy: def best_trials(self) -> list[FrozenTrial]: """Return the trials that is not dominated by other trials. - .. seealso:: - - See `Study.best_trials`_ for details. - - .. _Study.best_trials: https://optuna.readthedocs.io/en/stable/reference/\ - generated/optuna.study.Study.html#optuna.study.Study.best_trials - Returns: A list of FrozenTrial object """ @@ -182,6 +175,38 @@ class PreferentialStudy: """ self._study.add_trials(trials) + def enqueue_trial( + self, + params: dict[str, Any], + user_attrs: dict[str, Any] | None = None, + skip_if_exists: bool = False, + ) -> None: + """Enqueue a trial with given parameter values. + + You can fix the next sampling parameters which will be evaluated in your + objective function. + + .. seealso:: + + See `Study.enqueue_trials`_ for details. + + .. _Study.get_trials: https://optuna.readthedocs.io/en/stable/reference/\ + generated/optuna.study.Study.html#optuna.study.Study.enqueue_trials + + Args: + params: + Parameter values to pass your objective function. + user_attrs: + A dictionary of user-specific attributes other than ``params``. + skip_if_exists: + When :obj:`True`, prevents duplicate trials from being enqueued again. + + .. note:: + This method might produce duplicated trials if called simultaneously + by multiple processes at the same time with same ``params`` dict. + """ + self._study.enqueue_trial(params, user_attrs, skip_if_exists) + def report_preference( self, better_trials: FrozenTrial | list[FrozenTrial], @@ -219,8 +244,11 @@ class PreferentialStudy: Returns: A list of the pair of FrozenTrial objects. The left trial is better than the right one. """ + + preferences = get_preferences( + self._study._storage.get_study_system_attrs(self._study._study_id) + ) # Must come before study.get_trials() trials = self._study.get_trials(deepcopy=deepcopy) - preferences = get_preferences(self._study._study_id, self._study._storage) return [(trials[better], trials[worse]) for (better, worse) in preferences] def set_user_attr(self, key: str, value: Any) -> None: @@ -237,24 +265,6 @@ class PreferentialStudy: """ self._study.set_user_attr(key, value) - def mark_comparison_ready(self, trial_or_number: optuna.Trial | int) -> None: - """Mark trials ready to compare. - - Args: - trial_or_number: - A Trial object or trial_number. - """ - storage = self._study._storage - if isinstance(trial_or_number, optuna.Trial): - trial_id = trial_or_number._trial_id - elif isinstance(trial_or_number, int): - trial_id = storage.get_trial_id_from_study_id_trial_number( - self._study._study_id, trial_or_number - ) - else: - raise RuntimeError("Unexpected trial type") - storage.set_trial_system_attr(trial_id, _SYSTEM_ATTR_COMPARISON_READY, True) - def should_generate(self) -> bool: """Return whether the generator should generate a new trial now. @@ -263,21 +273,33 @@ class PreferentialStudy: to generate a new trial if this method returns :obj:`True`, and to wait for human evaluation if this method returns :obj:`False`. """ - return len(self.best_trials) < get_n_generate(self._study.system_attrs) + study_system_attrs = self._study._storage.get_study_system_attrs( + self._study._study_id + ) # Must come before _study.get_trials() + trials = self._study.get_trials( + deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) + ) + worse_trial_numbers = {worse for _, worse in get_preferences(study_system_attrs)} + skipped_trial_ids = set(get_skipped_trial_ids(study_system_attrs)) + active_trials = [ + t + for t in trials + if t.number not in worse_trial_numbers and t._trial_id not in skipped_trial_ids + ] + return len(active_trials) < get_n_generate(self._study.system_attrs) def get_best_trials(study_id: int, storage: optuna.storages.BaseStorage) -> list[FrozenTrial]: - preferences = get_preferences(study_id, storage) + preferences = get_preferences(storage.get_study_system_attrs(study_id)) worse_numbers = {worse for _, worse in preferences} + nondominated_numbers = {better for better, _ in preferences if better not in worse_numbers} + trials = storage.get_all_trials(study_id, deepcopy=False) + study_system_attrs = storage.get_study_system_attrs(study_id) + best_trials = [] - for t in storage.get_all_trials( - study_id, deepcopy=False, states=(TrialState.COMPLETE, TrialState.RUNNING) - ): - if not t.system_attrs.get(_SYSTEM_ATTR_COMPARISON_READY, False): - continue - if t.number in worse_numbers: - continue + for n in nondominated_numbers: + t = trials[n] if is_skipped_trial(t._trial_id, study_system_attrs): continue best_trials.append(copy.deepcopy(t)) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 2964c9e0..47c2a486 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -16,8 +16,9 @@ def report_preferences( study_id: int, storage: BaseStorage, preferences: list[tuple[int, int]], -) -> None: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + str(uuid.uuid4()) +) -> str: + preference_id = str(uuid.uuid4()) + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id storage.set_study_system_attr( study_id=study_id, key=key, @@ -31,15 +32,12 @@ def report_preferences( trial_id = trials[number]._trial_id if trials[number].state != TrialState.COMPLETE: storage.set_trial_state_values(trial_id, TrialState.COMPLETE, values) + return preference_id -def get_preferences( - study_id: int, - storage: BaseStorage, -) -> list[tuple[int, int]]: +def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]]: preferences: list[tuple[int, int]] = [] - system_attrs = storage.get_study_system_attrs(study_id) - for k, v in system_attrs.items(): + for k, v in study_system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_PREFERENCE): continue preferences.extend(v) # type: ignore @@ -63,6 +61,19 @@ def is_skipped_trial(trial_id: int, study_system_attrs: dict[str, Any]) -> bool: return key in study_system_attrs +def get_skipped_trial_ids(study_system_attrs: dict[str, Any]) -> list[int]: + skipped_trial_ids: list[int] = [] + for k in study_system_attrs: + if not k.startswith(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL): + continue + try: + trial_id = int(k[len(_SYSTEM_ATTR_PREFIX_SKIP_TRIAL) :]) # noqa: E203 + skipped_trial_ids.append(trial_id) + except ValueError: + continue + return skipped_trial_ids + + def get_n_generate(study_system_attrs: dict[str, Any]) -> int: return study_system_attrs[_SYSTEM_ATTR_N_GENERATE] diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index ff4001c1..8349b1a4 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,155 +1,41 @@ from __future__ import annotations import math -from math import erfc from typing import Any +from typing import Callable -from botorch.acquisition.analytic import LogExpectedImprovement -from botorch.models.gpytorch import GPyTorchModel -from botorch.optim import optimize_acqf +import botorch.acquisition.analytic +import botorch.models.model +import botorch.optim +import botorch.posteriors.gpytorch import gpytorch.constraints import gpytorch.kernels -import gpytorch.likelihoods.gaussian_likelihood -from gpytorch.likelihoods.gaussian_likelihood import GaussianLikelihood -from gpytorch.likelihoods.gaussian_likelihood import Interval from gpytorch.likelihoods.gaussian_likelihood import Prior -from gpytorch.models.exact_gp import ExactGP -import gpytorch.module -from linear_operator.operators import DiagLinearOperator -from linear_operator.operators import LinearOperator -from linear_operator.utils.errors import NotPSDError import numpy as np import optuna -from optuna import distributions -from optuna import Study -from optuna._transform import _SearchSpaceTransform -from optuna.distributions import BaseDistribution -from optuna.search_space import IntersectionSearchSpace -from optuna.trial import FrozenTrial -import pyro -import pyro.infer.autoguide -import pyro.infer.mcmc -from scipy.special import erfcinv +import optuna._transform import torch from torch import Tensor from .._system_attrs import get_preferences -class _WeightedGaussianLikelihood(GaussianLikelihood): - def __init__( - self, - weights: torch.Tensor | None = None, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, - batch_shape: torch.Size = torch.Size(), - **kwargs: Any, - ) -> None: - super().__init__( - noise_prior=noise_prior, - noise_constraint=noise_constraint, - batch_shape=batch_shape, - **kwargs, - ) - self.weights = weights - - def _shaped_noise_covar( - self, base_shape: torch.Size, *params: Any, **kwargs: Any - ) -> Tensor | LinearOperator: - assert self.weights is not None - assert base_shape[-1] == self.weights.shape[-1] - return DiagLinearOperator(1.0 / self.weights) * super()._shaped_noise_covar( - base_shape, *params, **kwargs - ) - - -def _sample_y( - preferences: np.ndarray, - cov_X_X: np.ndarray, - obs_noise_var: float, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: - # TODO: Refactor and write tests for this function. - - N = cov_X_X.shape[0] - M = len(preferences) - cov_X_X = cov_X_X + np.eye(N) * 1e-6 # Add jitter - cov_X_X_chol = np.linalg.cholesky(cov_X_X) - cov_X_X_inv = np.linalg.inv(cov_X_X) - - # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T - - schur = cov_X_X_inv.copy() - np.add.at(schur, (preferences[:, 0], preferences[:, 0]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 1]), 1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 0], preferences[:, 1]), -1.0 / (2 * obs_noise_var)) - np.add.at(schur, (preferences[:, 1], preferences[:, 0]), -1.0 / (2 * obs_noise_var)) - idx_M = np.arange(M) - - schur_inv = np.linalg.inv(schur) - - cov_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] - cov_diff_inv = cov_diff_inv[preferences[:, 0], :] - cov_diff_inv[preferences[:, 1], :] - cov_diff_inv *= -1 / (2 * obs_noise_var) ** 2 - cov_diff_inv[idx_M, idx_M] += 1.0 / (2 * obs_noise_var) - - diffs = _orthants_MVN_Gibbs_sampling( - cov_diff_inv, - cycles=cycles, - initial_sample=initial_sample[:, 0] - initial_sample[:, 1], - rng=rng, - )[-1] - - random_ys = (cov_X_X_chol @ rng.randn(N))[preferences] + np.sqrt(obs_noise_var) * rng.randn( - M, 2 - ) - errors = diffs - (random_ys[:, 0] - random_ys[:, 1]) - cov_diff_inv_errors = cov_diff_inv @ errors - - AT_cov_diff_inv_errors = np.zeros((N,)) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 0], cov_diff_inv_errors) - np.add.at(AT_cov_diff_inv_errors, preferences[:, 1], -cov_diff_inv_errors) - - return ( - random_ys - + (cov_X_X @ AT_cov_diff_inv_errors)[preferences] - + obs_noise_var * np.array([[1, -1]]) * cov_diff_inv_errors[:, None] - ) - - -_SQRT2 = math.sqrt(2) - - -def _orthants_MVN_Gibbs_sampling( - cov_inv: np.ndarray, - cycles: int, - initial_sample: np.ndarray, - rng: np.random.RandomState, -) -> np.ndarray: +def _orthants_MVN_Gibbs_sampling(cov_inv: Tensor, cycles: int, initial_sample: Tensor) -> Tensor: dim = cov_inv.shape[0] assert cov_inv.shape == (dim, dim) - if initial_sample is None: - sample_chain = np.zeros(dim) - else: - sample_chain = initial_sample + sample_chain = initial_sample + conditional_std = torch.rsqrt(torch.diag(cov_inv)) + scaled_cov_inv = cov_inv / torch.diag(cov_inv)[:, None] - conditional_std = 1 / np.sqrt(np.diag(cov_inv)) - - scaled_cov_inv = cov_inv / np.c_[np.diag(cov_inv)] - - out = np.empty((cycles + 1, dim)) + out = torch.empty((cycles + 1, dim), dtype=torch.float64) out[0, :] = sample_chain for i in range(cycles): for j in range(dim): conditional_mean = sample_chain[j] - scaled_cov_inv[j] @ sample_chain sample_chain[j] = ( - _one_side_trunc_norm_sampling( - lower=-conditional_mean / conditional_std[j], rng=rng - ) + _one_side_trunc_norm_sampling(lower=-conditional_mean / conditional_std[j]) * conditional_std[j] + conditional_mean ) @@ -158,144 +44,234 @@ def _orthants_MVN_Gibbs_sampling( return out -def _one_side_trunc_norm_sampling(lower: float, rng: np.random.RandomState) -> float: - return erfcinv(rng.rand() * erfc(lower / _SQRT2)) * _SQRT2 +def _one_side_trunc_norm_sampling(lower: Tensor) -> Tensor: + if lower > 4.0: + r = torch.clamp_min(torch.rand(torch.Size(()), dtype=torch.float64), min=1e-300) + return (lower * lower - 2 * r.log()).sqrt() + else: + SQRT2 = math.sqrt(2) + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + while 1 - r == 1: + r = torch.rand(torch.Size(()), dtype=torch.float64) * torch.erfc(lower / SQRT2) + return torch.erfinv(1 - r) * SQRT2 -class _PreferentialGP(GPyTorchModel, ExactGP): - _num_outputs = 1 +_orthants_MVN_Gibbs_sampling_jit = torch.jit.script(_orthants_MVN_Gibbs_sampling) + +def _compute_cov_diff_diff_inv(preferences: Tensor, cov_x_x: Tensor, noise_var: Tensor) -> Tensor: + N = cov_x_x.shape[0] + M = preferences.shape[0] + + # (sI + A K A^T)^-1 = s^-1 I - s^-2 A(K^-1 + s^-1 A^T A)^-1 A^T + # (K^-1 + s^-1 A^T A)^-1 = K (I + s^-1 A^T A K)^-1 (To avoid computing K^-1) + + I_plus_sinv_AT_A_K = torch.eye(N, dtype=torch.float64) + A_K = cov_x_x[preferences[:, 0], :] - cov_x_x[preferences[:, 1], :] + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 0], A_K * (1 / noise_var)) + I_plus_sinv_AT_A_K.index_add_(0, preferences[:, 1], A_K * (-1 / noise_var)) + schur_inv: Tensor = torch.linalg.solve(I_plus_sinv_AT_A_K, cov_x_x, left=False) + cov_diff_diff_inv = schur_inv[:, preferences[:, 0]] - schur_inv[:, preferences[:, 1]] + cov_diff_diff_inv = ( + cov_diff_diff_inv[preferences[:, 0], :] - cov_diff_diff_inv[preferences[:, 1], :] + ) + cov_diff_diff_inv *= -1 / noise_var**2 + idx_M = torch.arange(M) + cov_diff_diff_inv[idx_M, idx_M] += 1.0 / noise_var + + return cov_diff_diff_inv + + +class _SampledGP(botorch.models.model.Model): def __init__( self, - kernel: gpytorch.kernels.Kernel, - noise_prior: Prior | None = None, - noise_constraint: Interval | None = None, + kernel_func: Callable[[Tensor, Tensor], Tensor], + x: Tensor, + preferences: Tensor, + noise_var: Tensor, + diff: Tensor, ) -> None: - GPyTorchModel.__init__(self) - likelihood = _WeightedGaussianLikelihood( - noise_prior=noise_prior, noise_constraint=noise_constraint + super().__init__() + self.kernel_func = kernel_func + self.x = x + self.preferences = preferences + self.diff = diff + self.noise_var = noise_var + self._cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel_func(x, x), + noise_var=noise_var, ) - ExactGP.__init__(self, train_inputs=None, train_targets=None, likelihood=likelihood) - self.covar_module = kernel - self._last_params: dict[str, torch.Tensor] | None = None - self._last_mcmc_step_size: float | None = None + def posterior( + self, + X: Tensor, + output_indices: list[int] | None = None, + observation_noise: bool = False, + posterior_transform: Any | None = None, + **kwargs: Any, + ) -> botorch.posteriors.gpytorch.GPyTorchPosterior: + assert posterior_transform is None + assert output_indices is None + assert self.x.shape[-1] == X.shape[-1] - def _pyro_model(self, train_x: torch.Tensor, train_y: torch.Tensor) -> None: - # with gpytorch.settings.fast_computations(False, False, False): - sampled_model = self.pyro_sample_from_prior() + x_expanded = self.x.expand(X.shape[:-2] + (self.x.shape[-2], X.shape[-1])) - ys = sampled_model.likelihood(sampled_model.forward(train_x)) + cov_X_x = self.kernel_func(X, x_expanded) + cov_X_diff = cov_X_x[..., self.preferences[:, 0]] - cov_X_x[..., self.preferences[:, 1]] - pyro.sample("y", ys, obs=train_y) + mean = cov_X_diff @ (self._cov_diff_diff_inv @ self.diff) + cov = self.kernel_func(X, X) - cov_X_diff @ self._cov_diff_diff_inv @ cov_X_diff.transpose( + -1, -2 + ) + if observation_noise: + idx = torch.arange(cov.shape[-1]) + cov[..., idx, idx] += self.noise_var - def fit_mcmc( - self, X: torch.Tensor, preferences: torch.Tensor, cycles: int, rng: np.random.RandomState - ) -> None: + return botorch.posteriors.gpytorch.GPyTorchPosterior( + distribution=gpytorch.distributions.MultivariateNormal( + mean=mean, + covariance_matrix=cov, + ) + ) + + @property + def batch_shape(self) -> torch.Size: + return torch.Size() + + @property + def num_outputs(self) -> int: + return 1 + + +def _truncnorm_mean_var_logz(alpha: Tensor) -> tuple[Tensor, Tensor, Tensor]: + SQRT_HALF = math.sqrt(0.5) + SQRT_HALF_PI = math.sqrt(0.5 * math.pi) + logz = torch.special.log_ndtr(-alpha) + mean = 1 / (SQRT_HALF_PI * torch.special.erfcx(alpha * SQRT_HALF)) + var = 1 - mean * (mean - alpha) + return mean, var, logz + + +def _orthants_MVN_EP( + cov0: Tensor, preferences: Tensor, noise_var: Tensor, cycles: int +) -> tuple[Tensor, Tensor, Tensor]: + N = cov0.shape[0] + M = preferences.shape[0] + mu = torch.zeros(N, dtype=cov0.dtype) + cov = cov0.clone() + virtual_obs_a = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + virtual_obs_b = [torch.tensor(0.0, dtype=cov0.dtype) for _ in range(M)] + log_zs = torch.zeros(M, dtype=cov0.dtype) + + for _ in range(cycles): + for i in range(M): + pref_i = preferences[i, :] + mean1 = mu[pref_i[0]] - mu[pref_i[1]] + Sxy = cov[pref_i[0]] - cov[pref_i[1]] + var1 = Sxy[pref_i[0]] - Sxy[pref_i[1]] + + r0 = (1 - var1 * virtual_obs_a[i]).reciprocal() + var0 = var1 * r0 + mean0 = (mean1 + var1 * virtual_obs_b[i]) * r0 + + obs_var = var0 + noise_var + obs_sigma = torch.sqrt(obs_var) + alpha = -mean0 / torch.clamp_min(obs_sigma, min=1e-20) + mean_norm, var_norm, logz = _truncnorm_mean_var_logz(alpha) + + kalman_factor = var0 / torch.clamp_min(obs_var, min=1e-20) + mean2 = mean0 + obs_sigma * mean_norm * kalman_factor + var2 = kalman_factor * (noise_var + var_norm * var0) + + var1_var2_inv = torch.clamp_min(var1 * var2, min=1e-20).reciprocal() + db = (mean1 * var2 - mean2 * var1) * var1_var2_inv + da = (var1 - var2) * var1_var2_inv + virtual_obs_b[i] = virtual_obs_b[i] + db + virtual_obs_a[i] = virtual_obs_a[i] + da + + dr = (1 + var1 * da).reciprocal() + mu = mu - Sxy * ((db + mean1 * da) * dr) + cov = cov - (Sxy[:, None] * (da * dr)) @ Sxy[None, :] + log_zs[i] = logz + return mu, cov, torch.sum(log_zs) + + +_orthants_MVN_EP_jit = torch.jit.script(_orthants_MVN_EP) + + +class _PreferentialGP: + def __init__(self, kernel: gpytorch.kernels.Kernel, noise_prior: Prior, dims: int) -> None: + self.kernel = kernel + self.noise_prior = noise_prior + self.dims = dims + + self.diff = torch.empty((0,), dtype=torch.float64, requires_grad=False) + self.log_noise = torch.nn.Parameter( + torch.tensor(0.0, dtype=torch.float64), requires_grad=True + ) + + def fit_params_EP(self, X: Tensor, preferences: Tensor) -> None: if len(preferences) == 0: - # Skip actual MCMC computation - self.set_train_data( - inputs=torch.empty((0, X.shape[-1])), - targets=torch.empty((0,)), - strict=False, - ) - self.likelihood.weights = torch.empty((0,)) - else: - dtype = torch.float64 + return + tolerance = 1e-3 + max_iter = 100 - cnt = torch.bincount(preferences.reshape(-1)) - mask = cnt > 0 - train_x = X[mask] - weights = cnt[mask] + optim = torch.optim.LBFGS([*self.kernel.parameters(), self.log_noise]) - assert isinstance(self.likelihood, _WeightedGaussianLikelihood) - self.likelihood.weights = weights + last_params = [p.detach().clone() for p in optim.param_groups[0]["params"]] + for _ in range(max_iter): - preferences_np = preferences.detach().numpy() + def closure() -> Tensor: + optim.zero_grad() + noise = self.log_noise.exp() + cov0 = self.kernel.forward(X, X).to_dense() + _, _, logz = _orthants_MVN_EP_jit(cov0, preferences, noise, cycles=2) - all_ys_np = np.zeros((len(preferences), 2)) - train_y = torch.zeros( - ( - len( - train_x, - ) - ), - dtype=dtype, + loss = -logz - self.noise_prior.log_prob(noise) + for _, _, prior, param, _ in self.kernel.named_priors(): + loss = loss - prior.log_prob(param(self.kernel)).sum() + + loss.backward() + return loss + + optim.step(closure) + + # Check for convergence + params = optim.param_groups[0]["params"] + for p_old, p_new in zip(last_params, params): + if torch.max(torch.abs(p_old - p_new)) > tolerance: + break + else: + break + last_params = [p.detach().clone() for p in params] + + def sample_gp(self, x: Tensor, preferences: Tensor) -> _SampledGP: + self.fit_params_EP(x, preferences) + + with torch.no_grad(): + cov_diff_diff_inv = _compute_cov_diff_diff_inv( + preferences=preferences, + cov_x_x=self.kernel(x, x).to_dense(), + noise_var=self.log_noise.exp(), ) - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, + original_diff_size = len(self.diff) + self.diff.resize_(len(preferences)) + self.diff[original_diff_size:] = 0.0 + + self.diff = _orthants_MVN_Gibbs_sampling_jit( + cov_inv=cov_diff_diff_inv, + initial_sample=self.diff, + cycles=20, + )[-1] + return _SampledGP( + kernel_func=lambda x1, x2: self.kernel(x1, x2).to_dense(), + x=x, + preferences=preferences, + noise_var=self.log_noise.exp(), + diff=self.diff, ) - warmup_steps = max(0, cycles - 2) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - - raw_params = self._last_params or nuts.initial_params - for i in range(cycles): - params = { - name: nuts.transforms[name].inv(value) for name, value in raw_params.items() - } - _set_params(self, params) - self.set_train_data(train_x, train_y, strict=False) - all_ys_np = _sample_y( - preferences=preferences_np, - cov_X_X=self.covar_module(train_x).detach().numpy(), - obs_noise_var=float(self.likelihood.noise_covar.noise), - cycles=10, - initial_sample=all_ys_np, - rng=rng, - ) - ys_sum_np = np.zeros((len(X),)) - np.add.at(ys_sum_np, preferences_np.reshape(-1), all_ys_np.reshape(-1)) - ys_sum = torch.from_numpy(ys_sum_np) - train_y[:] = ys_sum[mask] / cnt[mask] - nuts.clear_cache() - try: - raw_params = nuts.sample(raw_params) - except NotPSDError: - nuts.cleanup() - nuts = pyro.infer.mcmc.NUTS( - model=self._pyro_model, - init_strategy=pyro.infer.autoguide.init_to_sample, - step_size=self._last_mcmc_step_size or 1.0, - ) - nuts.setup(warmup_steps=warmup_steps, train_x=train_x, train_y=train_y) - raw_params = nuts.initial_params - - params = {name: nuts.transforms[name].inv(value) for name, value in raw_params.items()} - self.set_train_data(train_x, train_y, strict=False) - _set_params(self, params) - - self._last_params = raw_params - self._last_mcmc_step_size = nuts.step_size - nuts.cleanup() - - def forward(self, x: torch.Tensor) -> gpytorch.distributions.MultivariateNormal: - mean_module = gpytorch.means.ZeroMean() - return gpytorch.distributions.MultivariateNormal( - mean_module(x), - self.covar_module(x), - ) - - -def _set_params( - module: gpytorch.Module, - params_dict: dict[str, torch.Tensor], - memo: set | None = None, - prefix: str = "", -) -> None: - if memo is None: - memo = set() - if hasattr(module, "_priors"): - for name, (prior, closure, setting_closure) in module._priors.items(): - if prior is not None and prior not in memo: - memo.add(prior) - setting_closure(module, params_dict[prefix + ("." if prefix else "") + name]) - - for mname, module_ in module.named_children(): - submodule_prefix = prefix + ("." if prefix else "") + mname - _set_params(module_, params_dict, memo=memo, prefix=submodule_prefix) class PreferentialGPSampler(optuna.samplers.BaseSampler): @@ -306,18 +282,16 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): noise_prior: Prior | None = None, independent_sampler: optuna.samplers.BaseSampler | None = None, seed: int | None = None, - device: torch.device | None = None, ) -> None: - self._rng = np.random.RandomState(seed) - self._search_space = IntersectionSearchSpace() - self.kernel = kernel - self.noise_prior = noise_prior - self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( - seed=self._rng.randint(2**32), - ) - self.device = device or torch.device("cpu") + self.noise_prior = noise_prior or gpytorch.priors.GammaPrior(5.0, 50.0) + self._rng = np.random.RandomState(seed) + self.independent_sampler = independent_sampler or optuna.samplers.RandomSampler( + seed=self._rng.randint(2**32) + ) + + self._search_space = optuna.search_space.IntersectionSearchSpace() self._gp: _PreferentialGP | None = None def reseed_rng(self) -> None: @@ -325,75 +299,64 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): self._rng = np.random.RandomState() def infer_relative_search_space( - self, study: Study, trial: FrozenTrial - ) -> dict[str, BaseDistribution]: + self, study: optuna.Study, trial: optuna.trial.FrozenTrial + ) -> dict[str, optuna.distributions.BaseDistribution]: return self._search_space.calculate(study) def sample_relative( self, - study: Study, - trial: FrozenTrial, - search_space: dict[str, BaseDistribution], + study: optuna.Study, + trial: optuna.trial.FrozenTrial, + search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: + preferences = get_preferences(study.system_attrs) + if len(preferences) == 0: + return {} + + trials = study.get_trials(deepcopy=False) + trials_with_preference = list({t for (b, w) in preferences for t in (b, w)}) + ids = {t: i for i, t in enumerate(trials_with_preference)} + + trans = optuna._transform._SearchSpaceTransform( + search_space, transform_log=True, transform_step=True, transform_0_1=True + ) + params = torch.tensor( + np.array([trans.transform(trials[t].params) for t in trials_with_preference]), + dtype=torch.float64, + ) + pref_ids = torch.tensor([[ids[b], ids[w]] for b, w in preferences], dtype=torch.int32) with torch.random.fork_rng(): torch.manual_seed(self._rng.randint(2**32)) - pyro.set_rng_seed(self._rng.randint(2**32)) - if len(search_space) == 0: - return {} - - preferences = get_preferences(study._study_id, study._storage) - trials = study.get_trials(deepcopy=False) - if len(preferences) == 0: - return {} - - trans = _SearchSpaceTransform( - search_space, transform_log=True, transform_step=True, transform_0_1=True - ) - dims = len(trans.bounds) self._gp = self._gp or _PreferentialGP( kernel=self.kernel or gpytorch.kernels.MaternKernel( - nu=2.5, - ard_num_dims=dims, - lengthscale_prior=gpytorch.priors.GammaPrior(3.0, 6.0), - lengthscale_constraint=gpytorch.constraints.Positive(), + nu=1.5, + ard_num_dims=len(trans.bounds), + lengthscale_prior=gpytorch.priors.GammaPrior(5.0, 10.0), + lengthscale_constraint=gpytorch.constraints.GreaterThan( + 0.0, + transform=torch.exp, + inv_transform=torch.log, + ), ), - noise_prior=self.noise_prior or gpytorch.priors.GammaPrior(1.1, 2.0), - noise_constraint=gpytorch.constraints.Positive(), + noise_prior=self.noise_prior, + dims=len(trans.bounds), ) + if self._gp.dims != len(trans.bounds): + raise NotImplementedError( + "The search space has changed. " + "Dynamic search space is not supported in PreferentialGPSampler." + ) - ids: dict[int, int] = {} - params: list[torch.Tensor] = [] - pref_ids: list[tuple[int, int]] = [] - - for better, worse in preferences: - for t in (better, worse): - if t not in ids: - ids[t] = len(ids) - params.append(trans.transform(trials[t].params)) - pref_ids.append((ids[better], ids[worse])) - dtype = torch.float64 - - params_torch = torch.tensor(np.array(params), dtype=dtype, device=self.device) - pref_ids_torch = torch.tensor( - np.array(pref_ids), - dtype=torch.int32, - device=self.device, - ) - self._gp.fit_mcmc(params_torch, pref_ids_torch, cycles=10, rng=self._rng) - self._gp.eval() - scores = self._gp(params_torch).mean - - best_f = torch.max(scores) - - acqf = LogExpectedImprovement( - model=self._gp, - best_f=best_f, + sampled_gp = self._gp.sample_gp(params, pref_ids) + acqf = botorch.acquisition.analytic.LogExpectedImprovement( + model=sampled_gp, + best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) # TODO: Make it possible to apply it on categorical variables - candidates, _ = optimize_acqf( + candidates, _ = botorch.optim.optimize_acqf( acq_function=acqf, bounds=torch.from_numpy(trans.bounds.T), q=1, @@ -407,10 +370,10 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): def sample_independent( self, - study: Study, - trial: FrozenTrial, + study: optuna.Study, + trial: optuna.trial.FrozenTrial, param_name: str, - param_distribution: distributions.BaseDistribution, + param_distribution: optuna.distributions.BaseDistribution, ) -> Any: return self.independent_sampler.sample_independent( study, trial, param_name, param_distribution diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index f0c82e2b..8979d3e3 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -588,10 +588,10 @@ export const actionCreator = () => { const updatePreference = ( study_id: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ) => { - reportPreferenceAPI(study_id, best_trials, worst_trials).catch((err) => { + reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { variant: "error", diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index caa57716..6ae34403 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,6 +55,28 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } +interface PreferenceHistoryResponce { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string +} + +const convertPreferenceHistory = ( + res: PreferenceHistoryResponce +): PreferenceHistory => { + return { + id: res.id, + preference_id: res.preference_id, + candidates: res.candidates, + clicked: res.clicked, + feedback_mode: res.mode, + timestamp: new Date(res.timestamp), + } +} + interface StudyDetailResponse { name: string datetime_start: string @@ -70,8 +92,8 @@ interface StudyDetailResponse { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - feedback_component_type?: string - feedback_artifact_key?: string + preference_history?: PreferenceHistoryResponce[] + plotly_graph_objects: PlotlyGraphObject[] } export const getStudyDetailAPI = ( @@ -110,6 +132,10 @@ export const getStudyDetailAPI = ( feedback_component_type: res.data .feedback_component_type as FeedbackComponentType, feedback_artifact_key: res.data.feedback_artifact_key, + preference_history: res.data.preference_history?.map( + convertPreferenceHistory + ), + plotly_graph_objects: res.data.plotly_graph_objects, } }) } @@ -319,13 +345,14 @@ export const getParamImportances = ( export const reportPreferenceAPI = ( studyId: number, - best_trials: number[], - worst_trials: number[] + candidates: number[], + clicked: number ): Promise => { return axiosInstance .post(`/api/studies/${studyId}/preference`, { - best_trials: best_trials, - worst_trials: worst_trials, + candidates: candidates, + clicked: clicked, + mode: "ChooseWorst", }) .then(() => { return diff --git a/optuna_dashboard/ts/components/App.tsx b/optuna_dashboard/ts/components/App.tsx index 78015049..8adf8895 100644 --- a/optuna_dashboard/ts/components/App.tsx +++ b/optuna_dashboard/ts/components/App.tsx @@ -96,6 +96,15 @@ export const App: FC = () => { /> } /> + + } + /> } diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 0903d72b..446b362e 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -34,12 +34,19 @@ import GitHubIcon from "@mui/icons-material/GitHub" import OpenInNewIcon from "@mui/icons-material/OpenInNew" import QueryStatsIcon from "@mui/icons-material/QueryStats" import ThumbUpAltIcon from "@mui/icons-material/ThumbUpAlt" +import HistoryIcon from "@mui/icons-material/History" import { Switch } from "@mui/material" import { actionCreator } from "../action" const drawerWidth = 240 -export type PageId = "top" | "analytics" | "trialTable" | "trialList" | "note" +export type PageId = + | "top" + | "analytics" + | "trialTable" + | "trialList" + | "note" + | "preferenceHistory" const openedMixin = (theme: Theme): CSSObject => ({ width: drawerWidth, @@ -204,6 +211,28 @@ export const AppDrawer: FC<{ /> + {isPreferential && ( + + + + + + + + + )} = ({ artifact, urlPath, height }) => { + if (isThreejsArtifact(artifact)) { + return ( + + ) + } else if (artifact.mimetype.startsWith("audio")) { + return ( + + ) + } else if (artifact.mimetype.startsWith("image")) { + return ( + + ) + } + return +} diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx new file mode 100644 index 00000000..3bc5a750 --- /dev/null +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -0,0 +1,218 @@ +import React, { FC, useState } from "react" +import { + Typography, + Box, + useTheme, + Card, + CardContent, + CardActions, +} from "@mui/material" +import ClearIcon from "@mui/icons-material/Clear" +import IconButton from "@mui/material/IconButton" +import OpenInFullIcon from "@mui/icons-material/OpenInFull" +import Modal from "@mui/material/Modal" +import { red } from "@mui/material/colors" + +import { TrialListDetail } from "./TrialList" +import { MarkdownRenderer } from "./Note" +import { formatDate } from "../dateUtil" + +type TrialType = "worst" | "none" + +const CandidateTrial: FC<{ + trial: Trial + type: TrialType +}> = ({ trial, type }) => { + const theme = useTheme() + const trialWidth = 300 + const trialHeight = 300 + const [detailShown, setDetailShown] = useState(false) + + const cardComponentSx = { + padding: 0, + position: "relative", + overflow: "hidden", + "::before": {}, + } + if (type !== "none") { + cardComponentSx["::before"] = { + content: '""', + position: "absolute", + top: 0, + left: 0, + width: "100%", + height: "100%", + backgroundColor: theme.palette.mode === "dark" ? "white" : "black", + opacity: 0.2, + zIndex: 1, + transition: "opacity 0.3s ease-out", + } + } + + return ( + + + Trial {trial.number} + setDetailShown(true)} + aria-label="show detail" + > + + + + + + + + + {type === "worst" ? ( + + ) : null} + + setDetailShown(false)}> + + + false} + directions={[]} + objectiveNames={[]} + /> + + + + + ) +} + +const ChoiceTrials: FC<{ choice: PreferenceHistory; trials: Trial[] }> = ({ + choice, + trials, +}) => { + const theme = useTheme() + const worst_trials = new Set([choice.clicked]) + + return ( + + + {formatDate(choice.timestamp)} + + + {choice.candidates.map((trial_num, index) => ( + + ))} + + + ) +} + +export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ + studyDetail, +}) => { + if ( + studyDetail === null || + !studyDetail.is_preferential || + studyDetail.preference_history === undefined + ) { + return null + } + const theme = useTheme() + const preference_histories = [...studyDetail.preference_history] + + if (preference_histories.length === 0) { + return ( + + No feedback history + + ) + } + + return ( + + {preference_histories.reverse().map((choice) => ( + + ))} + + ) +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ec597a09..e68e7ec9 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -13,46 +13,23 @@ import { FormLabel, Modal, } from "@mui/material" +import IconButton from "@mui/material/IconButton" import OpenInFullIcon from "@mui/icons-material/OpenInFull" import ReplayIcon from "@mui/icons-material/Replay" import ClearIcon from "@mui/icons-material/Clear" -import IconButton from "@mui/material/IconButton" import SettingsIcon from "@mui/icons-material/Settings" +import FullscreenIcon from "@mui/icons-material/Fullscreen" import red from "@mui/material/colors/red" + import { actionCreator } from "../action" -import { MarkdownRenderer } from "./Note" +import { TrialListDetail } from "./TrialList" import { - TrialArtifactActions, - TrialArtifactContent, - TrialListDetail, -} from "./TrialList" - -const FeedbackContent: FC<{ - trial: Trial - artifact?: Artifact - componentId: FeedbackComponentType - width: string - minHeight: string -}> = ({ trial, artifact, componentId, width, minHeight }) => { - if (componentId === "Note") { - return - } - if (componentId === "Artifact") { - if (artifact === undefined) { - return null - } - return ( - - ) - } - - return null -} + isThreejsArtifact, + useThreejsArtifactModal, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" +import { MarkdownRenderer } from "./Note" +import { Details } from "@mui/icons-material" const ModalPage: FC<{ children: React.ReactNode @@ -73,7 +50,7 @@ const ModalPage: FC<{ maxHeight: "90%", margin: "auto", overflow: "hidden", - backgroundColor: theme.palette.mode === "dark" ? "black" : "white", + backgroundColor: theme.palette.background.default, borderRadius: theme.spacing(3), }} > @@ -91,177 +68,6 @@ const ModalPage: FC<{ ) } -const PreferentialTrial: FC<{ - trial?: Trial - studyDetail: StudyDetail - hideTrial: () => void -}> = ({ trial, studyDetail, hideTrial }) => { - const theme = useTheme() - const action = actionCreator() - const trialWidth = 400 - const trialHeight = 300 - const [detailShown, setDetailShown] = useState(false) - const [buttonHover, setButtonHover] = useState(false) - const componentId = studyDetail.feedback_component_type ?? "Note" - const artifactKey = studyDetail.feedback_artifact_key - const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value - const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) - - if (trial == undefined) { - return ( - - ) - } - const onFeedback = () => { - hideTrial() - const best_trials = studyDetail.best_trials - .map((t) => t.number) - .filter((t) => t !== trial.number) - action.updatePreference(trial.study_id, best_trials, [trial.number]) - } - - return ( - - - Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( - - {`(${artifact.filename})`} - - ) : null} - {componentId === "Artifact" && artifact !== undefined ? ( - - ) : null} - { - hideTrial() - action.skipPreferentialTrial(trial.study_id, trial.trial_id) - }} - aria-label="skip trial" - > - - - setDetailShown(true)} - aria-label="show detail" - > - - - - { - if (e.shiftKey) onFeedback() - }} - sx={{ - position: "relative", - padding: theme.spacing(2), - overflow: "hidden", - minHeight: theme.spacing(20), - }} - > - - - - - - - - { - setDetailShown(false) - }} - > - true} - directions={[]} - objectiveNames={[]} - /> - - - ) -} - const SettingsPage: FC<{ studyDetail: StudyDetail settingShown: boolean @@ -358,6 +164,29 @@ const SettingsPage: FC<{ ) } +const FeedbackContent: FC<{ + trial: Trial + artifact?: Artifact + componentId: FeedbackComponentType + width: string + minHeight: string + urlPath: string +}> = ({ trial, artifact, componentId, width, minHeight, urlPath }) => { + if (componentId === "Note") { + return + } + if (componentId === "Artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + + return null +} + type DisplayTrials = { numbers: number[] last_number: number @@ -366,16 +195,29 @@ type DisplayTrials = { export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { + const theme = useTheme() + const action = actionCreator() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + const runningTrials = + studyDetail?.trials.filter((t) => t.state === "Running") ?? [] + const activeTrials = runningTrials.concat(studyDetail?.best_trials ?? []) + + const [displayTrials, setDisplayTrials] = useState({ + numbers: activeTrials.map((t) => t.number), + last_number: Math.max(...activeTrials.map((t) => t.number), -1), + }) + const [settingShown, setSettingShown] = useState(false) + const [detailTrial, setDetailTrial] = useState(null) + const [buttonHover, setButtonHover] = useState(null) + + const trialWidth = 400 + const trialHeight = 300 + if (studyDetail === null || !studyDetail.is_preferential) { return null } - const theme = useTheme() - const [displayTrials, setDisplayTrials] = useState({ - numbers: studyDetail.best_trials.map((t) => t.number), - last_number: Math.max(...studyDetail.best_trials.map((t) => t.number), -1), - }) - const [settingShown, setSettingShown] = useState(false) - const new_trails = studyDetail.best_trials.filter( + const new_trails = activeTrials.filter( (t) => displayTrials.last_number < t.number && displayTrials.numbers.find((n) => n === t.number) === undefined @@ -441,22 +283,209 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ Which trial is the worst? - {displayTrials.numbers.map((t, index) => ( - trial.number === t)} - studyDetail={studyDetail} - hideTrial={() => { - hideTrial(t) - }} - /> - ))} + {displayTrials.numbers.map((t, index) => { + const trial = activeTrials.find((trial) => trial.number === t) + const candidates = displayTrials.numbers.filter((n) => n !== -1) + const componentId = studyDetail.feedback_component_type ?? "Note" + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial?.user_attrs.find( + (a) => a.key === artifactKey + )?.value + const artifact = trial?.artifacts.find( + (a) => a.artifact_id === artifactId + ) + const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + + if (trial == undefined) { + return ( + + ) + } + + const is3dModel = + componentId === "Artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) + const onFeedback = () => { + hideTrial(trial.number) + action.updatePreference(trial.study_id, candidates, trial.number) + } + + return ( + + + + Trial {trial.number} + {componentId === "Artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + hideTrial(trial.number) + action.skipPreferentialTrial(trial.study_id, trial.trial_id) + }} + aria-label="skip trial" + > + + + setDetailTrial(trial.number)} + aria-label="show detail" + > + + + + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + ) + })} + {detailTrial !== null && ( + { + setDetailTrial(null) + }} + > + + studyDetail.trials.find((t) => t.trial_id === trialId)?.state === + "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + )} + {renderThreejsArtifactModal()} ) } diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index fedfd625..15cb7cf8 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -30,6 +30,7 @@ import { GraphEdf } from "./GraphEdf" import { TrialList } from "./TrialList" import { StudyHistory } from "./StudyHistory" import { PreferentialTrials } from "./PreferentialTrials" +import { PreferenceHistory } from "./PreferenceHistory" import { PreferentialAnalytics } from "./PreferentialAnalytics" interface ParamTypes { @@ -175,6 +176,8 @@ export const StudyDetail: FC<{ /> ) + } else if (page == "preferenceHistory") { + content = } const toolbar = ( diff --git a/optuna_dashboard/ts/components/StudyHistory.tsx b/optuna_dashboard/ts/components/StudyHistory.tsx index b47c557a..907acd1a 100644 --- a/optuna_dashboard/ts/components/StudyHistory.tsx +++ b/optuna_dashboard/ts/components/StudyHistory.tsx @@ -15,6 +15,7 @@ import { GraphIntermediateValues } from "./GraphIntermediateValues" import Grid2 from "@mui/material/Unstable_Grid2" import { DataGrid, DataGridColumn } from "./DataGrid" import { GraphHyperparameterImportance } from "./GraphHyperparameterImportances" +import { UserDefinedPlot } from "./UserDefinedPlot" import { BestTrialsCard } from "./BestTrialsCard" import { useStudyDetailValue, @@ -124,6 +125,16 @@ export const StudyHistory: FC<{ studyId: number }> = ({ studyId }) => { + {studyDetail !== null && + studyDetail.plotly_graph_objects.map((go) => ( + + + + + + + + ))} diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index 536f1d91..a88b9d2c 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -1,10 +1,17 @@ import * as THREE from "three" -import React, { useEffect, useState } from "react" +import React, { useEffect, useState, ReactNode } from "react" import { Canvas } from "@react-three/fiber" import { GizmoHelper, GizmoViewport, OrbitControls } from "@react-three/drei" import { STLLoader } from "three/examples/jsm/loaders/STLLoader" import { Rhino3dmLoader } from "three/examples/jsm/loaders/3DMLoader" import { PerspectiveCamera } from "three" +import { Modal, Box } from "@mui/material" + +export const isThreejsArtifact = (artifact: Artifact): boolean => { + return ( + artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") + ) +} interface ThreejsArtifactViewerProps { src: string @@ -109,3 +116,48 @@ export const ThreejsArtifactViewer: React.FC = ( ) } + +export const useThreejsArtifactModal = (): [ + (path: string, artifact: Artifact) => void, + () => ReactNode +] => { + const [open, setOpen] = useState(false) + const [target, setTarget] = useState<[string, Artifact | null]>(["", null]) + + const openModal = (artifactUrlPath: string, artifact: Artifact) => { + setTarget([artifactUrlPath, artifact]) + setOpen(true) + } + + const renderDeleteStudyDialog = () => { + return ( + { + setOpen(false) + setTarget(["", null]) + }} + > + + + + + ) + } + return [openModal, renderDeleteStudyDialog] +} diff --git a/optuna_dashboard/ts/components/TrialArtifactCards.tsx b/optuna_dashboard/ts/components/TrialArtifactCards.tsx new file mode 100644 index 00000000..3e15f7ed --- /dev/null +++ b/optuna_dashboard/ts/components/TrialArtifactCards.tsx @@ -0,0 +1,233 @@ +import React, { + ChangeEventHandler, + DragEventHandler, + FC, + MouseEventHandler, + useRef, + useState, +} from "react" +import { + Typography, + Box, + useTheme, + IconButton, + Card, + CardContent, + CardActionArea, +} from "@mui/material" +import UploadFileIcon from "@mui/icons-material/UploadFile" +import DownloadIcon from "@mui/icons-material/Download" +import DeleteIcon from "@mui/icons-material/Delete" +import FullscreenIcon from "@mui/icons-material/Fullscreen" + +import { actionCreator } from "../action" +import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" +import { + useThreejsArtifactModal, + isThreejsArtifact, +} from "./ThreejsArtifactViewer" +import { ArtifactCardMedia } from "./ArtifactCardMedia" + +export const TrialArtifactCards: FC<{ trial: Trial }> = ({ trial }) => { + const theme = useTheme() + const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = + useDeleteArtifactDialog() + const [openThreejsArtifactModal, renderThreejsArtifactModal] = + useThreejsArtifactModal() + + const width = "200px" + const height = "150px" + + return ( + <> + + Artifacts + + + {trial.artifacts.map((artifact) => { + const urlPath = `/artifacts/${trial.study_id}/${trial.trial_id}/${artifact.artifact_id}` + return ( + + + + + {artifact.filename} + + {isThreejsArtifact(artifact) ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + openDeleteArtifactDialog( + trial.study_id, + trial.trial_id, + artifact + ) + }} + > + + + + + + + + ) + })} + + + {renderDeleteArtifactDialog()} + {renderThreejsArtifactModal()} + + ) +} + +const TrialArtifactUploader: FC<{ + trial: Trial + width: string + height: string +}> = ({ trial, width, height }) => { + const theme = useTheme() + const action = actionCreator() + const [dragOver, setDragOver] = useState(false) + + if (trial.state !== "Running" && trial.state !== "Waiting") { + return null + } + const inputRef = useRef(null) + const handleClick: MouseEventHandler = () => { + if (!inputRef || !inputRef.current) { + return + } + inputRef.current.click() + } + const handleOnChange: ChangeEventHandler = (e) => { + const files = e.target.files + if (files === null) { + return + } + action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) + } + const handleDrop: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + const files = e.dataTransfer.files + setDragOver(false) + for (let i = 0; i < files.length; i++) { + action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) + } + } + const handleDragOver: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(true) + } + const handleDragLeave: DragEventHandler = (e) => { + e.stopPropagation() + e.preventDefault() + e.dataTransfer.dropEffect = "copy" + setDragOver(false) + } + return ( + + + + + + Upload a New File + + Drag your file here or click to browse. + + + + + ) +} diff --git a/optuna_dashboard/ts/components/TrialList.tsx b/optuna_dashboard/ts/components/TrialList.tsx index bb70c119..6922003a 100644 --- a/optuna_dashboard/ts/components/TrialList.tsx +++ b/optuna_dashboard/ts/components/TrialList.tsx @@ -1,13 +1,4 @@ -import React, { - ChangeEventHandler, - DragEventHandler, - FC, - MouseEventHandler, - ReactNode, - useMemo, - useRef, - useState, -} from "react" +import React, { FC, ReactNode, useMemo } from "react" import { Typography, Box, @@ -16,13 +7,7 @@ import { IconButton, Menu, MenuItem, - Card, - CardContent, - CardMedia, - CardActionArea, - Modal, } from "@mui/material" -import { SxProps } from "@mui/system" import Chip from "@mui/material/Chip" import Divider from "@mui/material/Divider" import List from "@mui/material/List" @@ -33,11 +18,6 @@ import ListSubheader from "@mui/material/ListSubheader" import FilterListIcon from "@mui/icons-material/FilterList" import CheckBoxOutlineBlankIcon from "@mui/icons-material/CheckBoxOutlineBlank" import CheckBoxIcon from "@mui/icons-material/CheckBox" -import UploadFileIcon from "@mui/icons-material/UploadFile" -import DownloadIcon from "@mui/icons-material/Download" -import DeleteIcon from "@mui/icons-material/Delete" -import FullscreenIcon from "@mui/icons-material/Fullscreen" -import InsertDriveFileIcon from "@mui/icons-material/InsertDriveFile" import StopCircleIcon from "@mui/icons-material/StopCircle" import { TrialNote } from "./Note" @@ -46,9 +26,8 @@ import ListItemIcon from "@mui/material/ListItemIcon" import { useRecoilValue } from "recoil" import { artifactIsAvailable } from "../state" import { actionCreator } from "../action" -import { useDeleteArtifactDialog } from "./DeleteArtifactDialog" import { TrialFormWidgets } from "./TrialFormWidgets" -import { ThreejsArtifactViewer } from "./ThreejsArtifactViewer" +import { TrialArtifactCards } from "./TrialArtifactCards" const states: TrialState[] = [ "Complete", @@ -320,344 +299,11 @@ export const TrialListDetail: FC<{ value !== null ? renderInfo(key, value) : null )} - {artifactEnabled && } + {artifactEnabled && } ) } -export const TrialArtifactContent: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - if (artifact.mimetype.startsWith("image")) { - return ( - - ) - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - - - - ) - } else if (artifact.mimetype.startsWith("audio")) { - return ( - - - - ) - } else { - return ( - - - - ) - } -} - -export const TrialArtifactActions: FC<{ - trial: Trial - artifact: Artifact - sx: SxProps -}> = ({ trial, artifact, sx }) => { - const [open3dModelViewer, setOpen3dModelViewer] = useState(false) - - if (artifact.mimetype.startsWith("image")) { - return null - } else if ( - artifact.filename.endsWith(".stl") || - artifact.filename.endsWith(".3dm") - ) { - return ( - <> - { - setOpen3dModelViewer(true) - }} - > - - - { - setOpen3dModelViewer(false) - }} - > - - - - - - ) - } - return null -} - -const TrialArtifact: FC<{ - trial: Trial - artifact: Artifact - width: string - height: string -}> = ({ trial, artifact, width, height }) => { - const [openDeleteArtifactDialog, renderDeleteArtifactDialog] = - useDeleteArtifactDialog() - const theme = useTheme() - const is3dModel = - artifact.filename.endsWith(".stl") || artifact.filename.endsWith(".3dm") - const canDelete = trial.state === "Running" || trial.state === "Waiting" - let actionsCount = 1 - if (canDelete) actionsCount += 1 - if (is3dModel) actionsCount += 1 - const actionsWidth = theme.spacing(actionsCount * 4) - - return ( - - - - - {artifact.filename} - - {is3dModel ? ( - - ) : null} - {canDelete ? ( - { - openDeleteArtifactDialog(trial.study_id, trial.trial_id, artifact) - }} - > - - - ) : null} - - - - - {renderDeleteArtifactDialog()} - - ) -} - -const TrialArtifacts: FC<{ trial: Trial }> = ({ trial }) => { - const theme = useTheme() - const action = actionCreator() - const [dragOver, setDragOver] = useState(false) - - const width = "200px" - const height = "150px" - - const inputRef = useRef(null) - const handleClick: MouseEventHandler = () => { - if (!inputRef || !inputRef.current) { - return - } - inputRef.current.click() - } - const handleOnChange: ChangeEventHandler = (e) => { - const files = e.target.files - if (files === null) { - return - } - action.uploadArtifact(trial.study_id, trial.trial_id, files[0]) - } - const handleDrop: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - const files = e.dataTransfer.files - setDragOver(false) - for (let i = 0; i < files.length; i++) { - action.uploadArtifact(trial.study_id, trial.trial_id, files[i]) - } - } - const handleDragOver: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(true) - } - const handleDragLeave: DragEventHandler = (e) => { - e.stopPropagation() - e.preventDefault() - e.dataTransfer.dropEffect = "copy" - setDragOver(false) - } - - return ( - <> - - Artifacts - - - {trial.artifacts.map((a) => ( - - ))} - {trial.state === "Running" || trial.state === "Waiting" ? ( - - - - - - Upload a New File - - Drag your file here or click to browse. - - - - - ) : null} - - - ) -} - const getTrialListLink = ( studyId: number, exclude: TrialState[], diff --git a/optuna_dashboard/ts/components/UserDefinedPlot.tsx b/optuna_dashboard/ts/components/UserDefinedPlot.tsx new file mode 100644 index 00000000..029c4b57 --- /dev/null +++ b/optuna_dashboard/ts/components/UserDefinedPlot.tsx @@ -0,0 +1,21 @@ +import * as plotly from "plotly.js-dist-min" +import React, { FC, useEffect } from "react" +import { Box } from "@mui/material" + +export const UserDefinedPlot: FC<{ + graphObject: PlotlyGraphObject +}> = ({ graphObject }) => { + const plotDomId = `user-defined-plot:${graphObject.id}` + + useEffect(() => { + try { + const parsed = JSON.parse(graphObject.graph_object) + plotly.react(plotDomId, parsed.data, parsed.layout) + } catch (e) { + // Avoid to crash the whole page when given invalid grpah objects. + console.error(e) + } + }, [graphObject]) + + return +} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 241ea310..66fc06ed 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -12,6 +12,7 @@ type TrialIntermediateValueNumber = number | "inf" | "-inf" | "nan" type TrialState = "Running" | "Complete" | "Pruned" | "Fail" | "Waiting" type TrialStateFinished = "Complete" | "Fail" | "Pruned" type StudyDirection = "maximize" | "minimize" | "not_set" +type PreferenceFeedbackMode = "ChooseWorst" type FeedbackComponentType = "Note" | "Artifact" type FloatDistribution = { @@ -182,6 +183,11 @@ type FormWidgets = widgets: UserAttrFormWidget[] } +type PlotlyGraphObject = { + id: string + graph_object: string +} + type StudyDetail = { id: number name: string @@ -200,6 +206,8 @@ type StudyDetail = { form_widgets?: FormWidgets feedback_component_type?: FeedbackComponentType feedback_artifact_key?: string + preference_history?: PreferenceHistory[] + plotly_graph_objects: PlotlyGraphObject[] } type StudyDetails = { @@ -209,3 +217,12 @@ type StudyDetails = { type StudyParamImportance = { [study_id: string]: ParamImportance[][] } + +type PreferenceHistory = { + id: string + preference_id: string + candidates: number[] + clicked: number + feedback_mode: PreferenceFeedbackMode + timestamp: Date +} diff --git a/pyproject.toml b/pyproject.toml index 0555f701..9b7ce731 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ docs = [ test = [ "coverage", + "plotly", "pytest", "moto[s3]", ] diff --git a/python_tests/preferential/test_study.py b/python_tests/preferential/test_study.py index 3de57dae..7fc1aaba 100644 --- a/python_tests/preferential/test_study.py +++ b/python_tests/preferential/test_study.py @@ -40,7 +40,6 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli for _ in range(2): trial = study.ask() trial.suggest_float("x", 0, 1) - study.mark_comparison_ready(trial) better, worse = study.trials study.report_preference(better, worse) assert len(study.preferences) == 1 @@ -152,7 +151,6 @@ def test_copy_study() -> None: for _ in range(3): trial = from_study.ask() trial.suggest_float("x", 0, 1) - from_study.mark_comparison_ready(trial) from_study.report_preference(from_study.trials[0], from_study.trials[1]) from_study.report_preference(from_study.trials[1], from_study.trials[2]) @@ -243,7 +241,6 @@ def test_get_trials(storage_supplier: Callable[[], StorageSupplier]) -> None: for _ in range(5): trial = study.ask() trial.suggest_int("x", 1, 5) - study.mark_comparison_ready(trial) with patch("copy.deepcopy", wraps=copy.deepcopy) as mock_object: trials0 = study.get_trials(deepcopy=False) @@ -266,8 +263,7 @@ def test_get_trials_state_option(storage_supplier: Callable[[], StorageSupplier] with storage_supplier() as storage: study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() better, worse = study.trials[:2] study.report_preference(better, worse) diff --git a/python_tests/preferential/test_system_attrs.py b/python_tests/preferential/test_system_attrs.py index 10448d48..34f93200 100644 --- a/python_tests/preferential/test_system_attrs.py +++ b/python_tests/preferential/test_system_attrs.py @@ -18,12 +18,13 @@ def test_report_and_get_preferences(storage_supplier: Callable[[], StorageSuppli study.ask() study_id = study._study_id - assert len(get_preferences(study_id, storage)) == 0 + + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 0 better, worse = study.trials[0], study.trials[1] report_preferences(study_id, storage, [(better.number, worse.number)]) - assert len(get_preferences(study_id, storage)) == 1 + assert len(get_preferences(storage.get_study_system_attrs(study_id))) == 1 - actual_better, actual_worse = get_preferences(study_id, storage)[0] + actual_better, actual_worse = get_preferences(storage.get_study_system_attrs(study_id))[0] assert actual_better == better.number assert actual_worse == worse.number diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 7633cd8f..29e4da9f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -105,10 +105,11 @@ class APITestCase(TestCase): storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() study.report_preference(study.trials[0], study.trials[1]) + assert len(study.best_trials) == 1 + app = create_app(storage) study_id = study._study._study_id status, _, body = send_request( @@ -120,16 +121,14 @@ class APITestCase(TestCase): self.assertEqual(status, 200) best_trials = json.loads(body)["best_trials"] - assert len(best_trials) == 2 + assert len(best_trials) == 1 assert best_trials[0]["number"] == 0 - assert best_trials[1]["number"] == 2 def test_report_preference(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -137,7 +136,13 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference", "POST", - body=json.dumps({"best_trials": [0, 2], "worst_trials": [1]}), + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), content_type="application/json", ) self.assertEqual(status, 204) @@ -152,13 +157,35 @@ class APITestCase(TestCase): assert better.number == 2 assert worse.number == 1 + def test_report_preference_when_typo_mode(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChoseWorst", + "candidates": [0, 1, 2], + "clicked": 1, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 400) + def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) register_output_component(study, "Note") for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id @@ -220,23 +247,23 @@ class APITestCase(TestCase): trials: list[optuna.Trial] = [] for _ in range(3): trial = study.ask() - study.mark_comparison_ready(trial) trials.append(trial) + study.report_preference(study.trials[0], study.trials[1]) + study.report_preference(study.trials[2], study.trials[1]) app = create_app(storage) study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/{trials[1]._trial_id}/skip", + f"/api/studies/{study_id}/{trials[0]._trial_id}/skip", "POST", content_type="application/json", ) self.assertEqual(status, 204) best_trials = study.best_trials - assert len(best_trials) == 2 - assert best_trials[0].number == 0 - assert best_trials[1].number == 2 + assert len(best_trials) == 1 + assert best_trials[0].number == 2 def test_create_study(self) -> None: for name, directions, expected_status in [ diff --git a/python_tests/test_custom_plot_data.py b/python_tests/test_custom_plot_data.py new file mode 100644 index 00000000..3dcfc856 --- /dev/null +++ b/python_tests/test_custom_plot_data.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import optuna +from optuna_dashboard import _custom_plot_data as custom_plot_data +from optuna_dashboard import save_plotly_graph_object +import pytest + + +def get_dummy_study() -> optuna.Study: + def objective(trial: optuna.Trial) -> float: + x = trial.suggest_float("x", -100, 100) + y = trial.suggest_categorical("y", [-1, 0, 1]) + return x**2 + y + + study = optuna.create_study() + optuna.logging.set_verbosity(optuna.logging.ERROR) + study.optimize(objective, n_trials=100) + return study + + +def test_save_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 2 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +def test_update_plotly_graph_object() -> None: + # Save history plot + dummy_study = get_dummy_study() + plot_data = optuna.visualization.plot_optimization_history(dummy_study) + graph_object_id = save_plotly_graph_object(dummy_study, plot_data) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + # Save parallel coordinate plot + plot_data = optuna.visualization.plot_parallel_coordinate(dummy_study) + graph_object_id = save_plotly_graph_object( + dummy_study, plot_data, graph_object_id=graph_object_id + ) + + study_system_attrs = dummy_study._storage.get_study_system_attrs(dummy_study._study_id) + plot_data_dict = custom_plot_data.get_plotly_graph_objects(study_system_attrs) + assert len(plot_data_dict) == 1 + assert plot_data_dict[graph_object_id] == plot_data.to_json() + + +@pytest.mark.parametrize( + "name", + [ + "0", + "a", + "a1-:_.", + ], +) +def test_is_valid_graph_object_id(name: str) -> None: + assert custom_plot_data.is_valid_graph_object_id(name) + + +@pytest.mark.parametrize( + "name", + [ + "a,", + "a b", + "aあいうえお", + ], +) +def test_is_invalid_graph_object_id(name: str) -> None: + assert not custom_plot_data.is_valid_graph_object_id(name) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py new file mode 100644 index 00000000..51c9b0f8 --- /dev/null +++ b/python_tests/test_preferential_history.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from typing import Callable + +from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._serializer import serialize_preference_history +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE + +from .storage_supplier import parametrize_storages +from .storage_supplier import StorageSupplier + + +@parametrize_storages +def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + + study_id = study._study._study_id + + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=1, + ), + ) + report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory( + mode="ChooseWorst", + candidates=[0, 2, 3, 4], + clicked=0, + ), + ) + history = serialize_preference_history(storage.get_study_system_attrs(study_id)) + sys_attrs = storage.get_study_system_attrs(study_id) + assert len(history) == 2 + assert history[0]["candidates"] == [0, 1, 2] + assert history[0]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + assert len(preferences) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst + assert history[1]["candidates"] == [0, 2, 3, 4] + assert history[1]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + assert len(preferences) == 3 + for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): + assert len(preferences[i]) == 2 + assert preferences[i][0] == best + assert preferences[i][1] == worst diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index a90e0de7..72db7b26 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,7 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert study_detail["is_preferential"] @@ -40,7 +40,7 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False) + study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) assert not study_detail["is_preferential"] From ded56df9fc54abe0cf34047f7e891a9ded690565 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 15:35:04 +0900 Subject: [PATCH 044/104] split component --- .../ts/components/PreferentialTrials.tsx | 377 +++++++++--------- 1 file changed, 196 insertions(+), 181 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index b7805259..b1ba6b2c 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -163,14 +163,12 @@ const SettingsPage: FC<{ ) } -const FeedbackContent: FC<{ +const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType - width: string - minHeight: string urlPath: string -}> = ({ trial, artifact, componentId, width, minHeight, urlPath }) => { +}> = ({ trial, artifact, componentId, urlPath }) => { if (componentId === "Note") { return } @@ -186,6 +184,191 @@ const FeedbackContent: FC<{ return null } +const PreferentialTrial: FC<{ + trial?: Trial + studyDetail: StudyDetail + candidates: number[] + hideTrial: () => void + openDetailTrial: () => void + openThreejsArtifactModal: (urlPath: string, artifact: Artifact) => void +}> = ({ + trial, + studyDetail, + candidates, + hideTrial, + openDetailTrial, + openThreejsArtifactModal, +}) => { + const theme = useTheme() + const action = actionCreator() + const [buttonHover, setButtonHover] = useState(false) + const trialWidth = 400 + const trialHeight = 300 + const componentId = studyDetail.feedback_component_type ?? "Note" + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + const is3dModel = + componentId === "Artifact" && + artifact !== undefined && + isThreejsArtifact(artifact) + + if (trial === undefined) { + return ( + + ) + } + + const onFeedback = () => { + hideTrial() + action.updatePreference(trial.study_id, candidates, trial.number) + } + + return ( + + + + Trial {trial.number} + {componentId === "Artifact" && artifact !== undefined ? ( + + {`(${artifact.filename})`} + + ) : null} + + {is3dModel ? ( + { + openThreejsArtifactModal(urlPath, artifact) + }} + > + + + ) : null} + { + hideTrial() + action.skipPreferentialTrial(trial.study_id, trial.trial_id) + }} + aria-label="skip trial" + > + + + + + + + { + if (e.shiftKey) onFeedback() + }} + sx={{ + position: "relative", + padding: theme.spacing(2), + overflow: "hidden", + minHeight: theme.spacing(20), + }} + > + + + + + + + ) +} + type DisplayTrials = { numbers: number[] last_number: number @@ -195,7 +378,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail, }) => { const theme = useTheme() - const action = actionCreator() const [openThreejsArtifactModal, renderThreejsArtifactModal] = useThreejsArtifactModal() const runningTrials = @@ -208,10 +390,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ }) const [settingShown, setSettingShown] = useState(false) const [detailTrial, setDetailTrial] = useState(null) - const [buttonHover, setButtonHover] = useState(null) - - const trialWidth = 400 - const trialHeight = 300 if (studyDetail === null || !studyDetail.is_preferential) { return null @@ -285,179 +463,16 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.numbers.map((t, index) => { const trial = activeTrials.find((trial) => trial.number === t) const candidates = displayTrials.numbers.filter((n) => n !== -1) - const componentId = studyDetail.feedback_component_type ?? "Note" - const artifactKey = studyDetail.feedback_artifact_key - const artifactId = trial?.user_attrs.find( - (a) => a.key === artifactKey - )?.value - const artifact = trial?.artifacts.find( - (a) => a.artifact_id === artifactId - ) - const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` - - if (trial == undefined) { - return ( - - ) - } - - const is3dModel = - componentId === "Artifact" && - artifact !== undefined && - isThreejsArtifact(artifact) - const onFeedback = () => { - hideTrial(trial.number) - action.updatePreference(trial.study_id, candidates, trial.number) - } - return ( - - - - Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( - - {`(${artifact.filename})`} - - ) : null} - - {is3dModel ? ( - { - openThreejsArtifactModal(urlPath, artifact) - }} - > - - - ) : null} - { - hideTrial(trial.number) - action.skipPreferentialTrial(trial.study_id, trial.trial_id) - }} - aria-label="skip trial" - > - - - setDetailTrial(trial.number)} - aria-label="show detail" - > - - - - { - if (e.shiftKey) onFeedback() - }} - sx={{ - position: "relative", - padding: theme.spacing(2), - overflow: "hidden", - minHeight: theme.spacing(20), - }} - > - - - - - - + hideTrial(t)} + openDetailTrial={() => setDetailTrial(t)} + openThreejsArtifactModal={openThreejsArtifactModal} + /> ) })} From d218818a72f1490a2e781a9ed5e33ba27b7d43e6 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 15:42:52 +0900 Subject: [PATCH 045/104] fix by lint --- python_tests/test_api.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 29e4da9f..e22d104c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -212,11 +212,10 @@ class APITestCase(TestCase): def test_change_component_type_only(self) -> None: storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage) + study = create_study(storage=storage, n_generate=3) register_output_component(study, "Artifact", "audio") for _ in range(3): - trial = study.ask() - study.mark_comparison_ready(trial) + study.ask() app = create_app(storage) study_id = study._study._study_id From a4d483fd0c199ade4bf570bd7281fbc8adb34584 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 16:03:48 +0900 Subject: [PATCH 046/104] fix by review --- optuna_dashboard/ts/components/AppDrawer.tsx | 11 +++-------- optuna_dashboard/ts/components/PreferentialGraph.tsx | 4 ++-- optuna_dashboard/ts/components/StudyDetail.tsx | 6 ++---- optuna_dashboard/ts/state.ts | 6 ++++++ 4 files changed, 13 insertions(+), 14 deletions(-) diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index e6422eb0..8740c68a 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -17,8 +17,7 @@ import ListItemText from "@mui/material/ListItemText" import { drawerOpenState, reloadIntervalState, - useStudyDetailValue, - useStudySummaryValue, + useStudyIsPreferencial, } from "../state" import { Link } from "react-router-dom" import AutoGraphIcon from "@mui/icons-material/AutoGraph" @@ -130,12 +129,8 @@ export const AppDrawer: FC<{ const action = actionCreator() const [open, setOpen] = useRecoilState(drawerOpenState) const reloadInterval = useRecoilValue(reloadIntervalState) - const studyDetail = - studyId !== undefined ? useStudyDetailValue(studyId) : null - const studySummary = - studyId !== undefined ? useStudySummaryValue(studyId) : null const isPreferential = - studyDetail?.is_preferential ?? studySummary?.is_preferential ?? false + studyId !== undefined ? useStudyIsPreferencial(studyId) : null const styleListItem = { display: "block", @@ -248,7 +243,7 @@ export const AppDrawer: FC<{ - {studyDetail?.is_preferential && ( + {isPreferential && ( { const preferences: [number, number][] = [] let n = 0 for (const [source, target] of input_preferences) { diff --git a/optuna_dashboard/ts/components/StudyDetail.tsx b/optuna_dashboard/ts/components/StudyDetail.tsx index 6db6c2ed..ab37d02b 100644 --- a/optuna_dashboard/ts/components/StudyDetail.tsx +++ b/optuna_dashboard/ts/components/StudyDetail.tsx @@ -18,8 +18,8 @@ import { actionCreator } from "../action" import { reloadIntervalState, useStudyDetailValue, + useStudyIsPreferencial, useStudyName, - useStudySummaryValue, } from "../state" import { TrialTable } from "./TrialTable" import { AppDrawer, PageId } from "./AppDrawer" @@ -52,11 +52,9 @@ export const StudyDetail: FC<{ const action = actionCreator() const studyId = useURLVars() const studyDetail = useStudyDetailValue(studyId) - const studySummary = useStudySummaryValue(studyId) const reloadInterval = useRecoilValue(reloadIntervalState) const studyName = useStudyName(studyId) - const isPreferential = - studySummary?.is_preferential ?? studyDetail?.is_preferential ?? false + const isPreferential = useStudyIsPreferencial(studyId) const title = studyName !== null ? `${studyName} (id=${studyId})` : `Study #${studyId}` diff --git a/optuna_dashboard/ts/state.ts b/optuna_dashboard/ts/state.ts index 44301bd1..98100d49 100644 --- a/optuna_dashboard/ts/state.ts +++ b/optuna_dashboard/ts/state.ts @@ -87,6 +87,12 @@ export const useStudyDirections = ( return studyDetail?.directions || studySummary?.directions || null } +export const useStudyIsPreferencial = (studyId: number): boolean | null => { + const studyDetail = useStudyDetailValue(studyId) + const studySummary = useStudySummaryValue(studyId) + return studyDetail?.is_preferential || studySummary?.is_preferential || null +} + export const useStudyName = (studyId: number): string | null => { const studyDetail = useStudyDetailValue(studyId) const studySummary = useStudySummaryValue(studyId) From b0ed82af7e779d4235e80babde6d0c28bc41c9b0 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 16:31:39 +0900 Subject: [PATCH 047/104] fix by review --- optuna_dashboard/_app.py | 22 +++++++------- optuna_dashboard/_preferential_history.py | 29 ++++++++++--------- optuna_dashboard/_serializer.py | 4 +-- .../preferential/_system_attrs.py | 4 +-- python_tests/test_api.py | 14 ++------- python_tests/test_preferential_history.py | 11 +++---- 6 files changed, 38 insertions(+), 46 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 65bb9f34..f497145e 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -29,8 +29,9 @@ from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preferential_history import NewHistory +from ._preferential_history import remove_history from ._preferential_history import report_history -from ._preferential_history import switching_history +from ._preferential_history import restore_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -307,18 +308,17 @@ def create_app( response.status = 204 return {} - @app.put("/api/studies//preference/") + @app.delete("/api/studies//preference/") @json_api_view - def switch_preference(study_id: int, history_uuid: str) -> dict[str, Any]: - try: - enable = request.json.get("enable", None) - if enable is None or not isinstance(enable, bool): - raise ValueError - except ValueError: - response.status = 400 - return {"reason": "Invalid request."} - switching_history(study_id, storage, history_uuid, enable) + def remove_preference(study_id: int, history_uuid: str) -> dict[str, Any]: + remove_history(study_id, storage, history_uuid) + response.status = 204 + return {} + @app.post("/api/studies//preference/") + @json_api_view + def restore_preference(study_id: int, history_uuid: str) -> dict[str, Any]: + restore_history(study_id, storage, history_uuid) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index a8fe2148..60fe6cb0 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -82,19 +82,20 @@ def report_history( return history_id -def switching_history(study_id: int, storage: BaseStorage, uuid: str, enable: bool) -> None: +def remove_history(study_id: int, storage: BaseStorage, uuid: str) -> None: system_attrs = storage.get_study_system_attrs(study_id) history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, "")) - if enable: - preferences = [ - (best, history["clicked"]) - for best in history["candidates"] - if best != history["clicked"] - ] - storage.set_study_system_attr( - study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], preferences - ) - else: # disable - storage.set_study_system_attr( - study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] - ) + storage.set_study_system_attr( + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] + ) + + +def restore_history(study_id: int, storage: BaseStorage, uuid: str) -> None: + system_attrs = storage.get_study_system_attrs(study_id) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, "")) + preferences = [ + (best, history["clicked"]) for best in history["candidates"] if best != history["clicked"] + ] + storage.set_study_system_attr( + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], preferences + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 35b9d6a7..9ce77fb3 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -18,7 +18,7 @@ from ._named_objectives import get_objective_names from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._system_attrs import is_preference_valid +from .preferential._system_attrs import is_preference_removed if TYPE_CHECKING: @@ -184,7 +184,7 @@ def serialize_preference_history( "timestamp": choice["timestamp"], "candidates": choice["candidates"], "clicked": choice["clicked"], - "enabled": is_preference_valid(system_attrs, choice["preference_id"]), + "is_removed": is_preference_removed(system_attrs, choice["preference_id"]), } histories.append(history) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 5ce83742..82ba61fa 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -44,10 +44,10 @@ def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]] return preferences -def is_preference_valid(study_system_attrs: dict[str, Any], uuid: str) -> bool: +def is_preference_removed(study_system_attrs: dict[str, Any], uuid: str) -> bool: key = _SYSTEM_ATTR_PREFIX_PREFERENCE + uuid preference = study_system_attrs.get(key, []) - return len(preference) > 0 + return len(preference) == 0 def report_skip( diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 22c60578..205bf966 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -234,12 +234,7 @@ class APITestCase(TestCase): status, _, _ = send_request( app, f"/api/studies/{study_id}/preference/{history_id}", - "PUT", - body=json.dumps( - { - "enable": False, - } - ), + "DELETE", content_type="application/json", ) self.assertEqual(status, 204) @@ -251,12 +246,7 @@ class APITestCase(TestCase): status, _, _ = send_request( app, f"/api/studies/{study_id}/preference/{history_id}", - "PUT", - body=json.dumps( - { - "enable": True, - } - ), + "POST", content_type="application/json", ) self.assertEqual(status, 204) diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 75e622db..3ba54fd4 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -6,8 +6,9 @@ from typing import TYPE_CHECKING from optuna_dashboard._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history -from optuna_dashboard._preferential_history import switching_history +from optuna_dashboard._preferential_history import restore_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -86,18 +87,18 @@ def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> N storage=storage, input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), ) - switching_history(study_id, storage, history_id, False) + remove_history(study_id, storage, history_id) preference, history = get_preferences_history(history_id) assert history["mode"] == "ChooseWorst" assert history["candidates"] == [0, 1, 2] assert history["clicked"] == 1 assert len(preference) == 0 - switching_history(study_id, storage, history_id, False) + remove_history(study_id, storage, history_id) preference, history = get_preferences_history(history_id) assert len(preference) == 0 - switching_history(study_id, storage, history_id, True) + restore_history(study_id, storage, history_id) preference, history = get_preferences_history(history_id) assert history["mode"] == "ChooseWorst" assert history["candidates"] == [0, 1, 2] @@ -108,6 +109,6 @@ def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> N assert preference[i][0] == best assert preference[i][1] == worst - switching_history(study_id, storage, history_id, True) + restore_history(study_id, storage, history_id) preference, history = get_preferences_history(history_id) assert len(preference) == 2 From 4a7c26fc9042f7d37360b4396db0984bc291128d Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 17:43:44 +0900 Subject: [PATCH 048/104] follow output component in history --- optuna_dashboard/ts/apiClient.ts | 7 ++++-- .../ts/components/PreferenceHistory.tsx | 23 +++++++++++++++++-- .../ts/components/PreferentialTrials.tsx | 17 +++++++++++--- optuna_dashboard/ts/types/index.d.ts | 2 +- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 6ae34403..53a23430 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -94,6 +94,8 @@ interface StudyDetailResponse { form_widgets?: FormWidgets preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] + feedback_component_type?: FeedbackComponentType + feedback_artifact_key?: string } export const getStudyDetailAPI = ( @@ -129,8 +131,9 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data - .feedback_component_type as FeedbackComponentType, + feedback_component_type: res.data.feedback_component_type + ? (res.data.feedback_component_type as FeedbackComponentType) + : "Note", feedback_artifact_key: res.data.feedback_artifact_key, preference_history: res.data.preference_history?.map( convertPreferenceHistory diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 3bc5a750..c290d7c6 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -14,8 +14,9 @@ import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" -import { MarkdownRenderer } from "./Note" +import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" import { formatDate } from "../dateUtil" +import { useStudyDetailValue } from "../state" type TrialType = "worst" | "none" @@ -26,8 +27,21 @@ const CandidateTrial: FC<{ const theme = useTheme() const trialWidth = 300 const trialHeight = 300 + const studyDetail = useStudyDetailValue(trial.study_id) const [detailShown, setDetailShown] = useState(false) + if (studyDetail === null) { + return null + } + const componentId = studyDetail.feedback_component_type + const artifactKey = studyDetail.feedback_artifact_key + const artifactId = trial.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + const cardComponentSx = { padding: 0, position: "relative", @@ -76,7 +90,12 @@ const CandidateTrial: FC<{ padding: theme.spacing(2), }} > - + {type === "worst" ? ( diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index b1ba6b2c..9e93a294 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -163,7 +163,7 @@ const SettingsPage: FC<{ ) } -const OutputContent: FC<{ +export const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId: FeedbackComponentType @@ -184,6 +184,14 @@ const OutputContent: FC<{ return null } +export const getArtifactUrlPath = ( + studyId: number, + trialId: number, + artifactId: string +) => { + return `/artifacts/${studyId}/${trialId}/${artifactId}` +} + const PreferentialTrial: FC<{ trial?: Trial studyDetail: StudyDetail @@ -204,11 +212,14 @@ const PreferentialTrial: FC<{ const [buttonHover, setButtonHover] = useState(false) const trialWidth = 400 const trialHeight = 300 - const componentId = studyDetail.feedback_component_type ?? "Note" + const componentId = studyDetail.feedback_component_type const artifactKey = studyDetail.feedback_artifact_key const artifactId = trial?.user_attrs.find((a) => a.key === artifactKey)?.value const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) - const urlPath = `/artifacts/${studyDetail.id}/${trial?.trial_id}/${artifact?.artifact_id}` + const urlPath = + trial !== undefined && artifactId !== undefined + ? getArtifactUrlPath(studyDetail.id, trial?.trial_id, artifactId) + : "" const is3dModel = componentId === "Artifact" && artifact !== undefined && diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 66fc06ed..88ece9a5 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -204,7 +204,7 @@ type StudyDetail = { is_preferential: boolean objective_names?: string[] form_widgets?: FormWidgets - feedback_component_type?: FeedbackComponentType + feedback_component_type: FeedbackComponentType feedback_artifact_key?: string preference_history?: PreferenceHistory[] plotly_graph_objects: PlotlyGraphObject[] From ee9ebafcf46a6c2cb2998df163e64f8ef4d62d36 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Mon, 11 Sep 2023 17:59:34 +0900 Subject: [PATCH 049/104] follow output component in graph --- .../ts/components/PreferentialGraph.tsx | 29 ++++++++++++++----- .../ts/components/PreferentialTrials.tsx | 6 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 48364598..a52c8571 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -1,4 +1,4 @@ -import React, { FC, useState, useCallback, useMemo, useEffect } from "react" +import React, { FC, useState, useCallback, useEffect } from "react" import { Card, CardContent, @@ -7,7 +7,6 @@ import { Box, Chip, } from "@mui/material" -import { MarkdownRenderer } from "./Note" import ReactFlow, { Node, NodeProps, @@ -24,6 +23,9 @@ import "reactflow/dist/style.css" import ELK from "elkjs/lib/elk.bundled.js" import { ElkNode } from "elkjs/lib/elk-api.js" +import { useStudyDetailValue } from "../state" +import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" + const elk = new ELK() const nodeWidth = 400 const nodeHeight = 300 @@ -39,10 +41,16 @@ const GraphNode: FC> = ({ data, isConnectable }) => { if (trial === undefined) { return null } - const noteBody = trial.note.body - const noteFC = useMemo(() => { - return - }, [noteBody]) + const studyDetail = useStudyDetailValue(trial.study_id) + const componentId = studyDetail?.feedback_component_type + const artifactKey = studyDetail?.feedback_artifact_key + const artifactId = trial.user_attrs.find((a) => a.key === artifactKey)?.value + const artifact = trial.artifacts.find((a) => a.artifact_id === artifactId) + const urlPath = + artifactId !== undefined + ? getArtifactUrlPath(trial.study_id, trial.trial_id, artifactId) + : "" + return ( > = ({ data, isConnectable }) => { style={{ background: "#555" }} isConnectable={isConnectable} /> - {noteFC} + + + = ({ trial, artifact, componentId, urlPath }) => { - if (componentId === "Note") { + if (componentId === undefined || componentId === "Note") { return } if (componentId === "Artifact") { @@ -188,7 +188,7 @@ export const getArtifactUrlPath = ( studyId: number, trialId: number, artifactId: string -) => { +): string => { return `/artifacts/${studyId}/${trialId}/${artifactId}` } From f9aa926a97e1f1481cc36f8169bd0f862b9c1106 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 11:41:18 +0900 Subject: [PATCH 050/104] fix by review --- optuna_dashboard/_app.py | 12 ++-- optuna_dashboard/_preferential_history.py | 40 +++++------ optuna_dashboard/_serializer.py | 3 +- .../preferential/_system_attrs.py | 4 +- optuna_dashboard/ts/action.ts | 23 ++++--- optuna_dashboard/ts/apiClient.ts | 23 ++++--- .../ts/components/PreferenceHistory.tsx | 44 ++++++++----- .../ts/components/PreferentialTrials.tsx | 66 +++++++++---------- optuna_dashboard/ts/types/index.d.ts | 3 +- python_tests/test_api.py | 6 +- python_tests/test_preferential_history.py | 12 ++-- 11 files changed, 125 insertions(+), 111 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index f497145e..a3ab595a 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -308,17 +308,17 @@ def create_app( response.status = 204 return {} - @app.delete("/api/studies//preference/") + @app.delete("/api/studies//preference/") @json_api_view - def remove_preference(study_id: int, history_uuid: str) -> dict[str, Any]: - remove_history(study_id, storage, history_uuid) + def remove_preference(study_id: int, history_id: str) -> dict[str, Any]: + remove_history(study_id, storage, history_id) response.status = 204 return {} - @app.post("/api/studies//preference/") + @app.post("/api/studies//preference/") @json_api_view - def restore_preference(study_id: int, history_uuid: str) -> dict[str, Any]: - restore_history(study_id, storage, history_uuid) + def restore_preference(study_id: int, history_id: str) -> dict[str, Any]: + restore_history(study_id, storage, history_id) response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 60fe6cb0..c8acf969 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -24,10 +24,10 @@ if TYPE_CHECKING: { "mode": FeedbackMode, "id": str, - "preference_id": str, "timestamp": str, "candidates": list[int], "clicked": int, + "preferences": list[tuple[int, int]], }, ) History = ChooseWorstHistory @@ -49,53 +49,45 @@ def report_history( # TODO(moririn): Use TypeGuard after adding other history types. if input_data.mode == "ChooseWorst": preferences = [ - (best, input_data.clicked) - for best in input_data.candidates - if best != input_data.clicked + (better, input_data.clicked) + for better in input_data.candidates + if better != input_data.clicked ] else: assert False, f"Unknown data: {input_data}" - preference_id = report_preferences( + id = report_preferences( study_id=study_id, storage=storage, preferences=preferences, ) - history_id = str(uuid.uuid4()) if input_data.mode == "ChooseWorst": history: ChooseWorstHistory = { "mode": "ChooseWorst", - "id": history_id, - "preference_id": preference_id, + "id": id, "timestamp": datetime.now().isoformat(), "candidates": input_data.candidates, "clicked": input_data.clicked, + "preferences": preferences, } - key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + key = _SYSTEM_ATTR_PREFIX_HISTORY + id storage.set_study_system_attr( study_id=study_id, key=key, value=json.dumps(history), ) - return history_id + return id -def remove_history(study_id: int, storage: BaseStorage, uuid: str) -> None: +def remove_history(study_id: int, storage: BaseStorage, id: str) -> None: + storage.set_study_system_attr(study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + id, []) + + +def restore_history(study_id: int, storage: BaseStorage, id: str) -> None: system_attrs = storage.get_study_system_attrs(study_id) - history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, "")) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) storage.set_study_system_attr( - study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] - ) - - -def restore_history(study_id: int, storage: BaseStorage, uuid: str) -> None: - system_attrs = storage.get_study_system_attrs(study_id) - history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + uuid, "")) - preferences = [ - (best, history["clicked"]) for best in history["candidates"] if best != history["clicked"] - ] - storage.set_study_system_attr( - study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], preferences + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["id"], history["preferences"] ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 9ce77fb3..d4ff5608 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -180,11 +180,10 @@ def serialize_preference_history( history = { "mode": "ChooseWorst", "id": choice["id"], - "preference_id": choice["preference_id"], "timestamp": choice["timestamp"], "candidates": choice["candidates"], "clicked": choice["clicked"], - "is_removed": is_preference_removed(system_attrs, choice["preference_id"]), + "is_removed": is_preference_removed(system_attrs, choice["id"]), } histories.append(history) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 82ba61fa..33d56f30 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -44,8 +44,8 @@ def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]] return preferences -def is_preference_removed(study_system_attrs: dict[str, Any], uuid: str) -> bool: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + uuid +def is_preference_removed(study_system_attrs: dict[str, Any], id: str) -> bool: + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + id preference = study_system_attrs.get(key, []) return len(preference) == 0 diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index f42a17bb..5b66c58f 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -16,7 +16,8 @@ import { deleteArtifactAPI, reportPreferenceAPI, skipPreferentialTrialAPI, - switchPreferentialHistoryAPI, + removePreferentialHistoryAPI, + restorePreferentialHistoryAPI, } from "./apiClient" import { graphVisibilityState, @@ -610,12 +611,17 @@ export const actionCreator = () => { }) } - const switchPreferentialHistory = ( - studyId: number, - historyUuid: string, - enable: boolean - ) => { - switchPreferentialHistoryAPI(studyId, historyUuid, enable).catch((err) => { + const removePreferentialHistory = (studyId: number, historyUuid: string) => { + removePreferentialHistoryAPI(studyId, historyUuid).catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to switch history. Reason: ${reason}`, { + variant: "error", + }) + console.log(err) + }) + } + const restorePreferentialHistory = (studyId: number, historyUuid: string) => { + restorePreferentialHistoryAPI(studyId, historyUuid).catch((err) => { const reason = err.response?.data.reason enqueueSnackbar(`Failed to switch history. Reason: ${reason}`, { variant: "error", @@ -645,7 +651,8 @@ export const actionCreator = () => { saveTrialUserAttrs, updatePreference, skipPreferentialTrial, - switchPreferentialHistory, + removePreferentialHistory, + restorePreferentialHistory, } } diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 5c9355b7..ecc4b7c2 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -57,12 +57,11 @@ const convertTrialResponse = (res: TrialResponse): Trial => { interface PreferenceHistoryResponce { id: string - preference_id: string candidates: number[] clicked: number mode: PreferenceFeedbackMode timestamp: string - enabled: boolean + is_removed: boolean } const convertPreferenceHistory = ( @@ -70,12 +69,11 @@ const convertPreferenceHistory = ( ): PreferenceHistory => { return { id: res.id, - preference_id: res.preference_id, candidates: res.candidates, clicked: res.clicked, feedback_mode: res.mode, timestamp: new Date(res.timestamp), - enabled: res.enabled, + isRemoved: res.is_removed, } } @@ -369,15 +367,22 @@ export const skipPreferentialTrialAPI = ( }) } -export const switchPreferentialHistoryAPI = ( +export const removePreferentialHistoryAPI = ( studyId: number, - historyUuid: string, - enable: boolean + historyUuid: string ): Promise => { return axiosInstance - .put(`/api/studies/${studyId}/preference/${historyUuid}`, { - enable: enable, + .delete(`/api/studies/${studyId}/preference/${historyUuid}`) + .then(() => { + return }) +} +export const restorePreferentialHistoryAPI = ( + studyId: number, + historyUuid: string +): Promise => { + return axiosInstance + .post(`/api/studies/${studyId}/preference/${historyUuid}`) .then(() => { return }) diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 0535dc2e..81f106bd 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -142,14 +142,10 @@ const ChoiceTrials: FC<{ trials: Trial[] study_id: number }> = ({ choice, trials, study_id }) => { - const [enabled, setEnabled] = useState(choice.enabled) + const [isRemoved, setRemoved] = useState(choice.isRemoved) const theme = useTheme() const worst_trials = new Set([choice.clicked]) const action = actionCreator() - const handleSwitch = () => { - setEnabled(!enabled) - action.switchPreferentialHistory(study_id, choice.id, !enabled) - } return ( {formatDate(choice.timestamp)} - - {choice.enabled ? : } - + {choice.isRemoved ? ( + { + setRemoved(false) + action.restorePreferentialHistory(study_id, choice.id) + }} + sx={{ + margin: `auto ${theme.spacing(2)}`, + }} + > + + + ) : ( + { + setRemoved(true) + action.removePreferentialHistory(study_id, choice.id) + }} + sx={{ + margin: `auto ${theme.spacing(2)}`, + }} + > + + + )} {choice.candidates.map((trial_num, index) => ( diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ab7f04ea..f079a80d 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -174,8 +174,8 @@ const PreferentialTrial: FC<{ } type DisplayTrials = { - numbers: number[] - last_number: number + display: number[] + clicked: number[] } export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ @@ -193,51 +193,55 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const activeTrials = runningTrials.concat(studyDetail.best_trials) const [displayTrials, setDisplayTrials] = useState({ - numbers: activeTrials.map((t) => t.number), - last_number: Math.max(...activeTrials.map((t) => t.number), -1), + display: [], + clicked: [], }) const new_trails = activeTrials.filter( (t) => - displayTrials.last_number < t.number && - displayTrials.numbers.find((n) => n === t.number) === undefined + !displayTrials.display.includes(t.number) && + !displayTrials.clicked.includes(t.number) ) if (new_trails.length > 0) { - setDisplayTrials((display) => { - const numbers = [...display.numbers] + setDisplayTrials((prev) => { + const display = [...prev.display] + const clicked = [...prev.clicked] new_trails.map((t) => { - const index = numbers.findIndex((n) => n === -1) + const index = display.findIndex((n) => n === -1) if (index === -1) { - numbers.push(t.number) + display.push(t.number) + clicked.push(-1) } else { - numbers[index] = t.number + display[index] = t.number } }) return { - numbers: numbers, - last_number: Math.max(...numbers, -1), + display: display, + clicked: clicked, } }) } const hideTrial = (num: number) => { - setDisplayTrials((display) => { - const index = display.numbers.findIndex((n) => n === num) + setDisplayTrials((prev) => { + const index = prev.display.findIndex((n) => n === num) if (index === -1) { - return display + return prev } - const numbers = [...displayTrials.numbers] - numbers[index] = -1 + const display = [...prev.display] + const clicked = [...prev.clicked] + display[index] = -1 + clicked[index] = num return { - numbers: numbers, - last_number: display.last_number, + display: display, + clicked: clicked, } }) } - const latestHistoryId = studyDetail?.preference_history - ?.filter((h) => h.enabled) - .pop()?.id + const latestHistoryId = + studyDetail?.preference_history?.filter((h) => !h.isRemoved).pop()?.id ?? + null if (undoHistoryId !== null && undoHistoryId !== latestHistoryId) { - setUndoHistoryId(null) + setUndoHistoryId(latestHistoryId) } return ( @@ -253,17 +257,13 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ Which trial is the worst? { - if (latestHistoryId === undefined) { + if (latestHistoryId === null) { return } setUndoHistoryId(latestHistoryId) - action.switchPreferentialHistory( - studyDetail.id, - latestHistoryId, - false - ) + action.removePreferentialHistory(studyDetail.id, latestHistoryId) }} sx={{ margin: "auto 0 auto auto", @@ -273,11 +273,11 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ - {displayTrials.numbers.map((t, index) => ( + {displayTrials.display.map((t, index) => ( trial.number === t)} - candidates={displayTrials.numbers.filter((n) => n !== -1)} + candidates={displayTrials.display.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index fd9ca113..014d2dfd 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -217,10 +217,9 @@ type StudyParamImportance = { type PreferenceHistory = { id: string - preference_id: string candidates: number[] clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date - enabled: boolean + isRemoved: boolean } diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 205bf966..6075ffa5 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -228,7 +228,7 @@ class APITestCase(TestCase): self.assertEqual(status, 204) histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 - assert histories[0]["enabled"] + assert not histories[0]["is_removed"] history_id = histories[0]["id"] status, _, _ = send_request( @@ -240,7 +240,7 @@ class APITestCase(TestCase): self.assertEqual(status, 204) histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 - assert not histories[0]["enabled"] + assert histories[0]["is_removed"] assert len(study.get_preferences()) == 0 status, _, _ = send_request( @@ -252,7 +252,7 @@ class APITestCase(TestCase): self.assertEqual(status, 204) histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 - assert histories[0]["enabled"] + assert not histories[0]["is_removed"] preferences = study.get_preferences() preferences.sort(key=lambda x: (x[0].number, x[1].number)) assert len(preferences) == 2 diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 3ba54fd4..c64f448c 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -46,7 +46,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2] assert history[0]["clicked"] == 1 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["id"]] assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): assert len(preferences[i]) == 2 @@ -54,7 +54,7 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst assert history[1]["candidates"] == [0, 2, 3, 4] assert history[1]["clicked"] == 0 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["id"]] assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): assert len(preferences[i]) == 2 @@ -72,13 +72,11 @@ def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> N study_id = study._study._study_id - def get_preferences_history(history_id: str) -> tuple[list[tuple[int, int]], History]: + def get_preferences_history(id: str) -> tuple[list[tuple[int, int]], History]: system_attrs = storage.get_study_system_attrs(study_id) - history: History = json.loads( - system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + history_id, "") - ) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) preference: list[tuple[int, int]] = system_attrs.get( - _SYSTEM_ATTR_PREFIX_PREFERENCE + history["preference_id"], [] + _SYSTEM_ATTR_PREFIX_PREFERENCE + id, [] ) return preference, history From ed054fb9ae0f4dd7b6a350b46820136df311919b Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 13:37:02 +0900 Subject: [PATCH 051/104] fix feedback screen --- optuna_dashboard/ts/components/PreferentialTrials.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index f079a80d..d4a06c54 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -241,7 +241,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail?.preference_history?.filter((h) => !h.isRemoved).pop()?.id ?? null if (undoHistoryId !== null && undoHistoryId !== latestHistoryId) { - setUndoHistoryId(latestHistoryId) + setUndoHistoryId(null) } return ( From bc4a76ea1f4f87565ebf1b4666a5f9e0e22d84b7 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 14:48:48 +0900 Subject: [PATCH 052/104] fix ui by review --- optuna_dashboard/ts/components/PreferenceHistory.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 81f106bd..9e19bf07 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -204,6 +204,7 @@ const ChoiceTrials: FC<{ flexDirection: "row", flexWrap: "wrap", filter: choice.isRemoved ? "brightness(0.4)" : undefined, + backgroundColor: theme.palette.background.paper, }} > {choice.candidates.map((trial_num, index) => ( From d88525a1bc7e0bf6cb00264ed1cece449fb4f566 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 14:49:04 +0900 Subject: [PATCH 053/104] fix by lint --- optuna_dashboard/_preferential_history.py | 1 - optuna_dashboard/_serializer.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index c8acf969..3d912190 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from datetime import datetime import json from typing import TYPE_CHECKING -import uuid from optuna.storages import BaseStorage diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 17c9518f..8388c736 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -18,8 +18,8 @@ from ._named_objectives import get_objective_names from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY -from .preferential._system_attrs import is_preference_removed from .preferential._system_attrs import get_preferences +from .preferential._system_attrs import is_preference_removed if TYPE_CHECKING: From b4a4c2dae0bdac885634836ad8ba9ee871f35ab3 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 15:28:24 +0900 Subject: [PATCH 054/104] split api part of preference feedback component --- optuna_dashboard/_app.py | 23 ++++++++++ optuna_dashboard/_preference_setting.py | 60 ++++++++++++++++++++++++ optuna_dashboard/_serializer.py | 6 +++ python_tests/test_api.py | 61 +++++++++++++++++++++++++ python_tests/test_preference_setting.py | 18 ++++++++ 5 files changed, 168 insertions(+) create mode 100644 optuna_dashboard/_preference_setting.py create mode 100644 python_tests/test_preference_setting.py diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index c32c2061..fad4b74c 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preference_setting import _register_output_component from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -306,6 +307,28 @@ def create_app( response.status = 204 return {} + @app.post("/api/studies//component") + @json_api_view + def post_component(study_id: int) -> dict[str, Any]: + try: + component_type = request.json.get("component_type", "") + artifact_key = request.json.get("artifact_key", None) + except ValueError: + response.status = 400 + return {"reason": "invalid request."} + if component_type not in ["Note", "Artifact"]: + response.status = 400 + return {"reason": "component_type must be either 'Note' or 'Artifact'."} + + _register_output_component( + study_id=study_id, + storage=storage, + component_type=component_type, + artifact_key=artifact_key, + ) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py new file mode 100644 index 00000000..93b1a34c --- /dev/null +++ b/optuna_dashboard/_preference_setting.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +from optuna.storages import BaseStorage + +from .preferential._study import PreferentialStudy + + +if TYPE_CHECKING: + from typing import Literal + + OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + +_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" +_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" + + +def _register_output_component( + study_id: int, + storage: BaseStorage, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str | None = None, +) -> None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, + value=component_type, + ) + if artifact_key is not None: + storage.set_study_system_attr( + study_id=study_id, + key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, + value=artifact_key, + ) + + +def register_output_component( + study: PreferentialStudy, + component_type: OUTPUT_COMPONENT_TYPE, + artifact_key: str = "", +) -> None: + """Register output component to the study. + + Args: + study: + The study to register the output component. + component_type: + The type of the output component. + artifact_key: + When the component_type is "Artifact", + this argument is used as the attribute key of the artifact. + Each trial displays the artifact whose id is the value of the attribute. + """ + _register_output_component( + study_id=study._study._study_id, + storage=study._study._storage, + component_type=component_type, + artifact_key=artifact_key, + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index e3f77649..511e7365 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,6 +15,8 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -162,6 +164,10 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: + serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] + if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: + serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c551e3f2..e22d104c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preference_setting import register_output_component from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -179,6 +180,66 @@ class APITestCase(TestCase): ) self.assertEqual(status, 400) + def test_change_component(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Note") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Artifact" + assert study_detail["feedback_artifact_key"] == "image" + + def test_change_component_type_only(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + register_output_component(study, "Artifact", "audio") + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/component", + "POST", + body=json.dumps({"component_type": "Note"}), + content_type="application/json", + ) + self.assertEqual(status, 204) + + status, _, body = send_request( + app, + f"/api/studies/{study_id}", + "GET", + content_type="application/json", + ) + self.assertEqual(status, 200) + + study_detail = json.loads(body) + assert study_detail["feedback_component_type"] == "Note" + assert study_detail["feedback_artifact_key"] == "audio" + def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(n_generate=4, storage=storage) diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py new file mode 100644 index 00000000..033a9092 --- /dev/null +++ b/python_tests/test_preference_setting.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +from unittest import TestCase + +import optuna +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard.preferential._study import PreferentialStudy + + +class FeedbackSettingTestCase(TestCase): + def test_widget_to_dict_from_dict(self) -> None: + study = PreferentialStudy(optuna.create_study()) + register_output_component(study, "Artifact", "image_key") + system_attrs = study._study.system_attrs + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" + assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" From 5fcc64cff3e44bb34fb7ab9f2296fd7a8b4ff4e4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:00:46 +0900 Subject: [PATCH 055/104] fix by review --- optuna_dashboard/_app.py | 10 +++--- optuna_dashboard/_preference_setting.py | 24 ++++++-------- optuna_dashboard/_serializer.py | 10 +++--- python_tests/test_api.py | 44 ++++--------------------- python_tests/test_preference_setting.py | 14 ++++---- 5 files changed, 34 insertions(+), 68 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index fad4b74c..2ad7874e 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_output_component +from ._preference_setting import _register_preference_feedback_component_type from ._preferential_history import NewHistory from ._preferential_history import report_history from ._rdb_migration import register_rdb_migration_route @@ -307,11 +307,11 @@ def create_app( response.status = 204 return {} - @app.post("/api/studies//component") + @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def post_component(study_id: int) -> dict[str, Any]: + def put_component(study_id: int) -> dict[str, Any]: try: - component_type = request.json.get("component_type", "") + component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) except ValueError: response.status = 400 @@ -320,7 +320,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_output_component( + _register_preference_feedback_component_type( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 93b1a34c..73e9b489 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -12,30 +12,26 @@ if TYPE_CHECKING: OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] -_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE = "preference:component_type" -_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY = "preference:component_artifact_key" +_SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_output_component( +def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str | None = None, + artifact_key: str = "", ) -> None: storage.set_study_system_attr( study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, - value=component_type, + key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, + value={ + "type": component_type, + "artifact_key": artifact_key, + } ) - if artifact_key is not None: - storage.set_study_system_attr( - study_id=study_id, - key=_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, - value=artifact_key, - ) -def register_output_component( +def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str = "", @@ -52,7 +48,7 @@ def register_output_component( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ - _register_output_component( + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 511e7365..98db5ffb 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -15,8 +15,7 @@ from optuna.trial import FrozenTrial from . import _note as note from ._form_widget import get_form_widgets_json from ._named_objectives import get_objective_names -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE +from ._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY @@ -164,10 +163,9 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets - if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: - serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] - if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: - serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] + if serialized["is_preferential"]: + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index e22d104c..1908c188 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Note") + register_preference_feedback_component_type(study, "Note") for _ in range(3): study.ask() @@ -191,9 +191,9 @@ class APITestCase(TestCase): study_id = study._study._study_id status, _, _ = send_request( app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Artifact", "artifact_key": "image"}), + f"/api/studies/{study_id}/preference_feedback_component_type", + "PUT", + body=json.dumps({"type": "Artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,38 +207,8 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Artifact" - assert study_detail["feedback_artifact_key"] == "image" - - def test_change_component_type_only(self) -> None: - storage = optuna.storages.InMemoryStorage() - study = create_study(storage=storage, n_generate=3) - register_output_component(study, "Artifact", "audio") - for _ in range(3): - study.ask() - - app = create_app(storage) - study_id = study._study._study_id - status, _, _ = send_request( - app, - f"/api/studies/{study_id}/component", - "POST", - body=json.dumps({"component_type": "Note"}), - content_type="application/json", - ) - self.assertEqual(status, 204) - - status, _, body = send_request( - app, - f"/api/studies/{study_id}", - "GET", - content_type="application/json", - ) - self.assertEqual(status, 200) - - study_detail = json.loads(body) - assert study_detail["feedback_component_type"] == "Note" - assert study_detail["feedback_artifact_key"] == "audio" + assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: storage = optuna.storages.InMemoryStorage() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 033a9092..8a601af7 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -3,16 +3,18 @@ from __future__ import annotations from unittest import TestCase import optuna -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY -from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE -from optuna_dashboard._preference_setting import register_output_component +from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT +from optuna_dashboard._preference_setting import register_preference_feedback_component_type from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_output_component(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "Artifact", "image_key") system_attrs = study._study.system_attrs - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE, "") == "Artifact" - assert system_attrs.get(_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY, "") == "image_key" + feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + assert "type" in feedback_type + assert feedback_type["type"] == "Artifact" + assert "artifact_key" in feedback_type + assert feedback_type["artifact_key"] == "image_key" From fffa586da3f197b8ab3f39243d428a75c0143075 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:04:40 +0900 Subject: [PATCH 056/104] fix by review --- optuna_dashboard/_preference_setting.py | 2 +- optuna_dashboard/_serializer.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 73e9b489..d9d7c1c2 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -27,7 +27,7 @@ def _register_preference_feedback_component_type( value={ "type": component_type, "artifact_key": artifact_key, - } + }, ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 98db5ffb..f9aeabec 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,7 +165,8 @@ def serialize_study_detail( serialized["form_widgets"] = form_widgets if serialized["is_preferential"]: serialized["feedback_component_type"] = system_attrs.get( - _SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) + _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} + ) if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) From 63ac4996e5120466f3ebaac6e384e41a7712b743 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:13:51 +0900 Subject: [PATCH 057/104] remove and restore history api --- optuna_dashboard/_app.py | 16 ++++ optuna_dashboard/_preferential_history.py | 34 ++++++--- optuna_dashboard/_serializer.py | 12 ++- .../preferential/_system_attrs.py | 6 ++ python_tests/test_api.py | 60 +++++++++++++++ python_tests/test_preferential_history.py | 75 ++++++++++++++++--- 6 files changed, 173 insertions(+), 30 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index c32c2061..a3ab595a 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -29,7 +29,9 @@ from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials from ._preferential_history import NewHistory +from ._preferential_history import remove_history from ._preferential_history import report_history +from ._preferential_history import restore_history from ._rdb_migration import register_rdb_migration_route from ._serializer import serialize_study_detail from ._serializer import serialize_study_summary @@ -306,6 +308,20 @@ def create_app( response.status = 204 return {} + @app.delete("/api/studies//preference/") + @json_api_view + def remove_preference(study_id: int, history_id: str) -> dict[str, Any]: + remove_history(study_id, storage, history_id) + response.status = 204 + return {} + + @app.post("/api/studies//preference/") + @json_api_view + def restore_preference(study_id: int, history_id: str) -> dict[str, Any]: + restore_history(study_id, storage, history_id) + response.status = 204 + return {} + @app.post("/api/trials//tell") @json_api_view def tell_trial(trial_id: int) -> dict[str, Any]: diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 6b81b9bb..3d912190 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -4,10 +4,10 @@ from dataclasses import dataclass from datetime import datetime import json from typing import TYPE_CHECKING -import uuid from optuna.storages import BaseStorage +from .preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE from .preferential._system_attrs import report_preferences @@ -23,10 +23,10 @@ if TYPE_CHECKING: { "mode": FeedbackMode, "id": str, - "preference_id": str, "timestamp": str, "candidates": list[int], "clicked": int, + "preferences": list[tuple[int, int]], }, ) History = ChooseWorstHistory @@ -43,38 +43,50 @@ def report_history( study_id: int, storage: BaseStorage, input_data: NewHistory, -) -> None: +) -> str: preferences = [] # TODO(moririn): Use TypeGuard after adding other history types. if input_data.mode == "ChooseWorst": preferences = [ - (best, input_data.clicked) - for best in input_data.candidates - if best != input_data.clicked + (better, input_data.clicked) + for better in input_data.candidates + if better != input_data.clicked ] else: assert False, f"Unknown data: {input_data}" - preference_id = report_preferences( + id = report_preferences( study_id=study_id, storage=storage, preferences=preferences, ) - history_id = str(uuid.uuid4()) if input_data.mode == "ChooseWorst": history: ChooseWorstHistory = { "mode": "ChooseWorst", - "id": history_id, - "preference_id": preference_id, + "id": id, "timestamp": datetime.now().isoformat(), "candidates": input_data.candidates, "clicked": input_data.clicked, + "preferences": preferences, } - key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + key = _SYSTEM_ATTR_PREFIX_HISTORY + id storage.set_study_system_attr( study_id=study_id, key=key, value=json.dumps(history), ) + return id + + +def remove_history(study_id: int, storage: BaseStorage, id: str) -> None: + storage.set_study_system_attr(study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + id, []) + + +def restore_history(study_id: int, storage: BaseStorage, id: str) -> None: + system_attrs = storage.get_study_system_attrs(study_id) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) + storage.set_study_system_attr( + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["id"], history["preferences"] + ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index e3f77649..8388c736 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -19,15 +19,13 @@ from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import get_preferences +from .preferential._system_attrs import is_preference_removed if TYPE_CHECKING: from typing import Literal from typing import TypedDict - from ._preferential_history import ChooseWorstHistory - from ._preferential_history import History - Attribute = TypedDict( "Attribute", { @@ -174,20 +172,20 @@ def serialize_study_detail( def serialize_preference_history( system_attrs: dict[str, Any], -) -> list[History]: - histories: list[History] = [] +) -> list[dict[str, Any]]: + histories: list[dict[str, Any]] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue choice: dict[str, Any] = json.loads(v) if choice["mode"] == "ChooseWorst": - history: ChooseWorstHistory = { + history = { "mode": "ChooseWorst", "id": choice["id"], - "preference_id": choice["preference_id"], "timestamp": choice["timestamp"], "candidates": choice["candidates"], "clicked": choice["clicked"], + "is_removed": is_preference_removed(system_attrs, choice["id"]), } histories.append(history) diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 47c2a486..33d56f30 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -44,6 +44,12 @@ def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]] return preferences +def is_preference_removed(study_system_attrs: dict[str, Any], id: str) -> bool: + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + id + preference = study_system_attrs.get(key, []) + return len(preference) == 0 + + def report_skip( study_id: int, trial_id: int, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index c551e3f2..6075ffa5 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from .wsgi_client import send_request @@ -203,6 +204,65 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0].number == 2 + def test_undo_redo_history(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference", + "POST", + body=json.dumps( + { + "mode": "ChooseWorst", + "candidates": [0, 1, 2], + "clicked": 2, + } + ), + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert not histories[0]["is_removed"] + + history_id = histories[0]["id"] + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference/{history_id}", + "DELETE", + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert histories[0]["is_removed"] + assert len(study.get_preferences()) == 0 + + status, _, _ = send_request( + app, + f"/api/studies/{study_id}/preference/{history_id}", + "POST", + content_type="application/json", + ) + self.assertEqual(status, 204) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert not histories[0]["is_removed"] + preferences = study.get_preferences() + preferences.sort(key=lambda x: (x[0].number, x[1].number)) + assert len(preferences) == 2 + better, worse = preferences[0] + assert better.number == 0 + assert worse.number == 2 + better, worse = preferences[1] + assert better.number == 1 + assert worse.number == 2 + def test_create_study(self) -> None: for name, directions, expected_status in [ ("single-objective success", ["minimize"], 201), diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index 51c9b0f8..c64f448c 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -1,9 +1,14 @@ from __future__ import annotations +import json from typing import Callable +from typing import TYPE_CHECKING +from optuna_dashboard._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history +from optuna_dashboard._preferential_history import restore_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential._system_attrs import _SYSTEM_ATTR_PREFIX_PREFERENCE @@ -12,6 +17,10 @@ from .storage_supplier import parametrize_storages from .storage_supplier import StorageSupplier +if TYPE_CHECKING: + from optuna_dashboard._preferential_history import History + + @parametrize_storages def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: @@ -25,27 +34,19 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) report_history( study_id=study_id, storage=storage, - input_data=NewHistory( - mode="ChooseWorst", - candidates=[0, 1, 2], - clicked=1, - ), + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), ) report_history( study_id=study_id, storage=storage, - input_data=NewHistory( - mode="ChooseWorst", - candidates=[0, 2, 3, 4], - clicked=0, - ), + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 2, 3, 4], clicked=0), ) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 assert history[0]["candidates"] == [0, 1, 2] assert history[0]["clicked"] == 1 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["preference_id"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["id"]] assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): assert len(preferences[i]) == 2 @@ -53,9 +54,59 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst assert history[1]["candidates"] == [0, 2, 3, 4] assert history[1]["clicked"] == 0 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["preference_id"]] + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["id"]] assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): assert len(preferences[i]) == 2 assert preferences[i][0] == best assert preferences[i][1] == worst + + +@parametrize_storages +def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + + study_id = study._study._study_id + + def get_preferences_history(id: str) -> tuple[list[tuple[int, int]], History]: + system_attrs = storage.get_study_system_attrs(study_id) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) + preference: list[tuple[int, int]] = system_attrs.get( + _SYSTEM_ATTR_PREFIX_PREFERENCE + id, [] + ) + return preference, history + + history_id = report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), + ) + remove_history(study_id, storage, history_id) + preference, history = get_preferences_history(history_id) + assert history["mode"] == "ChooseWorst" + assert history["candidates"] == [0, 1, 2] + assert history["clicked"] == 1 + assert len(preference) == 0 + + remove_history(study_id, storage, history_id) + preference, history = get_preferences_history(history_id) + assert len(preference) == 0 + + restore_history(study_id, storage, history_id) + preference, history = get_preferences_history(history_id) + assert history["mode"] == "ChooseWorst" + assert history["candidates"] == [0, 1, 2] + assert history["clicked"] == 1 + assert len(preference) == 2 + for i, (best, worst) in enumerate([(0, 1), (2, 1)]): + assert len(preference[i]) == 2 + assert preference[i][0] == best + assert preference[i][1] == worst + + restore_history(study_id, storage, history_id) + preference, history = get_preferences_history(history_id) + assert len(preference) == 2 From adbbea39533b67f952071c4ae5eea7d68cb73dd4 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 16:23:06 +0900 Subject: [PATCH 058/104] wip: change hiding system on feedback screen in order to undo history --- .../ts/components/PreferentialTrials.tsx | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 08efd9a7..dc6e6c95 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -173,8 +173,8 @@ const PreferentialTrial: FC<{ } type DisplayTrials = { - numbers: number[] - last_number: number + display: number[] + clicked: number[] } export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ @@ -189,64 +189,70 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const activeTrials = runningTrials.concat(studyDetail.best_trials) const [displayTrials, setDisplayTrials] = useState({ - numbers: activeTrials.map((t) => t.number), - last_number: Math.max(...activeTrials.map((t) => t.number), -1), + display: [], + clicked: [], }) const new_trails = activeTrials.filter( (t) => - displayTrials.last_number < t.number && - displayTrials.numbers.find((n) => n === t.number) === undefined + !displayTrials.display.includes(t.number) && + !displayTrials.clicked.includes(t.number) ) if (new_trails.length > 0) { - setDisplayTrials((display) => { - const numbers = [...display.numbers] + setDisplayTrials((prev) => { + const display = [...prev.display] + const clicked = [...prev.clicked] new_trails.map((t) => { - const index = numbers.findIndex((n) => n === -1) + const index = display.findIndex((n) => n === -1) if (index === -1) { - numbers.push(t.number) + display.push(t.number) + clicked.push(-1) } else { - numbers[index] = t.number + display[index] = t.number } }) return { - numbers: numbers, - last_number: Math.max(...numbers, -1), + display: display, + clicked: clicked, } }) } const hideTrial = (num: number) => { - setDisplayTrials((display) => { - const index = display.numbers.findIndex((n) => n === num) + setDisplayTrials((prev) => { + const index = prev.display.findIndex((n) => n === num) if (index === -1) { - return display + return prev } - const numbers = [...displayTrials.numbers] - numbers[index] = -1 + const display = [...prev.display] + const clicked = [...prev.clicked] + display[index] = -1 + clicked[index] = num return { - numbers: numbers, - last_number: display.last_number, + display: display, + clicked: clicked, } }) } return ( - - Which trial is the worst? - + + + Which trial is the worst? + + - {displayTrials.numbers.map((t, index) => ( + {displayTrials.display.map((t, index) => ( trial.number === t)} - candidates={displayTrials.numbers.filter((n) => n !== -1)} + candidates={displayTrials.display.filter((n) => n !== -1)} hideTrial={() => { hideTrial(t) }} From ba82fadad9d735684c9cf3879951407a835c6599 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 17:43:23 +0900 Subject: [PATCH 059/104] fix by review --- optuna_dashboard/_app.py | 15 ++++- optuna_dashboard/_preferential_history.py | 32 ++++++++-- optuna_dashboard/_serializer.py | 20 ++++-- .../preferential/_system_attrs.py | 4 +- python_tests/test_api.py | 48 ++++++++++---- python_tests/test_preferential_history.py | 64 +++++++++++++------ 6 files changed, 132 insertions(+), 51 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index a3ab595a..a020c8b8 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,6 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials +from ._preferential_history import HistoryIdError from ._preferential_history import NewHistory from ._preferential_history import remove_history from ._preferential_history import report_history @@ -311,14 +312,24 @@ def create_app( @app.delete("/api/studies//preference/") @json_api_view def remove_preference(study_id: int, history_id: str) -> dict[str, Any]: - remove_history(study_id, storage, history_id) + try: + remove_history(study_id, storage, history_id) + except HistoryIdError: + response.status = 404 + return {"reason": f"history_id={history_id} is not found"} + response.status = 204 return {} @app.post("/api/studies//preference/") @json_api_view def restore_preference(study_id: int, history_id: str) -> dict[str, Any]: - restore_history(study_id, storage, history_id) + try: + restore_history(study_id, storage, history_id) + except HistoryIdError: + response.status = 404 + return {"reason": f"history_id={history_id} is not found"} + response.status = 204 return {} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 3d912190..9726d27c 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -30,6 +30,17 @@ if TYPE_CHECKING: }, ) History = ChooseWorstHistory + SerializedHistory = TypedDict( + "SerializedHistory", + { + "history": History, + "is_removed": bool, + }, + ) + + +class HistoryIdError(Exception): + pass @dataclass @@ -80,13 +91,20 @@ def report_history( return id -def remove_history(study_id: int, storage: BaseStorage, id: str) -> None: - storage.set_study_system_attr(study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + id, []) - - -def restore_history(study_id: int, storage: BaseStorage, id: str) -> None: +def remove_history(study_id: int, storage: BaseStorage, history_id: str) -> None: system_attrs = storage.get_study_system_attrs(study_id) - history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) + history_key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + if history_key not in system_attrs: + raise HistoryIdError + storage.set_study_system_attr(study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, []) + + +def restore_history(study_id: int, storage: BaseStorage, history_id: str) -> None: + system_attrs = storage.get_study_system_attrs(study_id) + history_key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id + if history_key not in system_attrs: + raise HistoryIdError + history: History = json.loads(system_attrs.get(history_key, "")) storage.set_study_system_attr( - study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history["id"], history["preferences"] + study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, history["preferences"] ) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 8388c736..001f689a 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -26,6 +26,9 @@ if TYPE_CHECKING: from typing import Literal from typing import TypedDict + from ._preferential_history import History + from ._preferential_history import SerializedHistory + Attribute = TypedDict( "Attribute", { @@ -172,24 +175,29 @@ def serialize_study_detail( def serialize_preference_history( system_attrs: dict[str, Any], -) -> list[dict[str, Any]]: - histories: list[dict[str, Any]] = [] +) -> list[SerializedHistory]: + histories: list[SerializedHistory] = [] for k, v in system_attrs.items(): if not k.startswith(_SYSTEM_ATTR_PREFIX_HISTORY): continue choice: dict[str, Any] = json.loads(v) if choice["mode"] == "ChooseWorst": - history = { + history: History = { "mode": "ChooseWorst", "id": choice["id"], "timestamp": choice["timestamp"], "candidates": choice["candidates"], "clicked": choice["clicked"], - "is_removed": is_preference_removed(system_attrs, choice["id"]), + "preferences": choice["preferences"], } - histories.append(history) + histories.append( + { + "history": history, + "is_removed": is_preference_removed(system_attrs, choice["id"]), + } + ) - histories.sort(key=lambda c: datetime.fromisoformat(c["timestamp"])) + histories.sort(key=lambda c: datetime.fromisoformat(c["history"]["timestamp"])) return histories diff --git a/optuna_dashboard/preferential/_system_attrs.py b/optuna_dashboard/preferential/_system_attrs.py index 33d56f30..438b411c 100644 --- a/optuna_dashboard/preferential/_system_attrs.py +++ b/optuna_dashboard/preferential/_system_attrs.py @@ -44,8 +44,8 @@ def get_preferences(study_system_attrs: dict[str, Any]) -> list[tuple[int, int]] return preferences -def is_preference_removed(study_system_attrs: dict[str, Any], id: str) -> bool: - key = _SYSTEM_ATTR_PREFIX_PREFERENCE + id +def is_preference_removed(study_system_attrs: dict[str, Any], preference_id: str) -> bool: + key = _SYSTEM_ATTR_PREFIX_PREFERENCE + preference_id preference = study_system_attrs.get(key, []) return len(preference) == 0 diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 6075ffa5..afe5a31c 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,6 +8,9 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study +from optuna_dashboard._preferential_history import NewHistory +from optuna_dashboard._preferential_history import remove_history +from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history from optuna_dashboard.preferential import create_study @@ -204,7 +207,7 @@ class APITestCase(TestCase): assert len(best_trials) == 1 assert best_trials[0].number == 2 - def test_undo_redo_history(self) -> None: + def test_remove_history(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) for _ in range(3): @@ -212,25 +215,19 @@ class APITestCase(TestCase): app = create_app(storage) study_id = study._study._study_id - status, _, _ = send_request( - app, - f"/api/studies/{study_id}/preference", - "POST", - body=json.dumps( - { - "mode": "ChooseWorst", - "candidates": [0, 1, 2], - "clicked": 2, - } + history_id = report_history( + study_id, + storage, + NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=2, ), - content_type="application/json", ) - self.assertEqual(status, 204) histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) assert len(histories) == 1 assert not histories[0]["is_removed"] - history_id = histories[0]["id"] status, _, _ = send_request( app, f"/api/studies/{study_id}/preference/{history_id}", @@ -243,6 +240,29 @@ class APITestCase(TestCase): assert histories[0]["is_removed"] assert len(study.get_preferences()) == 0 + def test_restore_history(self) -> None: + storage = optuna.storages.InMemoryStorage() + study = create_study(storage=storage, n_generate=3) + for _ in range(3): + study.ask() + + app = create_app(storage) + study_id = study._study._study_id + history_id = report_history( + study_id, + storage, + NewHistory( + mode="ChooseWorst", + candidates=[0, 1, 2], + clicked=2, + ), + ) + remove_history(study_id, storage, history_id) + histories = serialize_preference_history(storage.get_study_system_attrs(study_id)) + assert len(histories) == 1 + assert histories[0]["is_removed"] + assert len(study.get_preferences()) == 0 + status, _, _ = send_request( app, f"/api/studies/{study_id}/preference/{history_id}", diff --git a/python_tests/test_preferential_history.py b/python_tests/test_preferential_history.py index c64f448c..3d1a7d70 100644 --- a/python_tests/test_preferential_history.py +++ b/python_tests/test_preferential_history.py @@ -4,6 +4,7 @@ import json from typing import Callable from typing import TYPE_CHECKING +from optuna.storages import BaseStorage from optuna_dashboard._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history @@ -44,17 +45,17 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) history = serialize_preference_history(storage.get_study_system_attrs(study_id)) sys_attrs = storage.get_study_system_attrs(study_id) assert len(history) == 2 - assert history[0]["candidates"] == [0, 1, 2] - assert history[0]["clicked"] == 1 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["id"]] + assert history[0]["history"]["candidates"] == [0, 1, 2] + assert history[0]["history"]["clicked"] == 1 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[0]["history"]["id"]] assert len(preferences) == 2 for i, (best, worst) in enumerate([(0, 1), (2, 1)]): assert len(preferences[i]) == 2 assert preferences[i][0] == best assert preferences[i][1] == worst - assert history[1]["candidates"] == [0, 2, 3, 4] - assert history[1]["clicked"] == 0 - preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["id"]] + assert history[1]["history"]["candidates"] == [0, 2, 3, 4] + assert history[1]["history"]["clicked"] == 0 + preferences = sys_attrs[_SYSTEM_ATTR_PREFIX_PREFERENCE + history[1]["history"]["id"]] assert len(preferences) == 3 for i, (best, worst) in enumerate([(2, 0), (3, 0), (4, 0)]): assert len(preferences[i]) == 2 @@ -62,42 +63,65 @@ def test_report_and_get_choices(storage_supplier: Callable[[], StorageSupplier]) assert preferences[i][1] == worst +def get_preferences_history( + study_id: int, + storage: BaseStorage, + history_id: str, +) -> tuple[list[tuple[int, int]], History]: + system_attrs = storage.get_study_system_attrs(study_id) + history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + history_id, "")) + preference: list[tuple[int, int]] = system_attrs.get( + _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, [] + ) + return preference, history + + @parametrize_storages -def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> None: +def test_remove_history(storage_supplier: Callable[[], StorageSupplier]) -> None: with storage_supplier() as storage: study = create_study(storage=storage, n_generate=5) for _ in range(5): trial = study.ask() trial.suggest_float("x", 0, 1) - study_id = study._study._study_id - def get_preferences_history(id: str) -> tuple[list[tuple[int, int]], History]: - system_attrs = storage.get_study_system_attrs(study_id) - history: History = json.loads(system_attrs.get(_SYSTEM_ATTR_PREFIX_HISTORY + id, "")) - preference: list[tuple[int, int]] = system_attrs.get( - _SYSTEM_ATTR_PREFIX_PREFERENCE + id, [] - ) - return preference, history - history_id = report_history( study_id=study_id, storage=storage, input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), ) remove_history(study_id, storage, history_id) - preference, history = get_preferences_history(history_id) + preference, history = get_preferences_history(study_id, storage, history_id) assert history["mode"] == "ChooseWorst" assert history["candidates"] == [0, 1, 2] assert history["clicked"] == 1 assert len(preference) == 0 remove_history(study_id, storage, history_id) - preference, history = get_preferences_history(history_id) + preference, history = get_preferences_history(study_id, storage, history_id) + assert len(preference) == 0 + + +@parametrize_storages +def test_restore_history(storage_supplier: Callable[[], StorageSupplier]) -> None: + with storage_supplier() as storage: + study = create_study(storage=storage, n_generate=5) + for _ in range(5): + trial = study.ask() + trial.suggest_float("x", 0, 1) + study_id = study._study._study_id + + history_id = report_history( + study_id=study_id, + storage=storage, + input_data=NewHistory(mode="ChooseWorst", candidates=[0, 1, 2], clicked=1), + ) + remove_history(study_id, storage, history_id) + preference, history = get_preferences_history(study_id, storage, history_id) assert len(preference) == 0 restore_history(study_id, storage, history_id) - preference, history = get_preferences_history(history_id) + preference, history = get_preferences_history(study_id, storage, history_id) assert history["mode"] == "ChooseWorst" assert history["candidates"] == [0, 1, 2] assert history["clicked"] == 1 @@ -108,5 +132,5 @@ def test_undo_redo_history(storage_supplier: Callable[[], StorageSupplier]) -> N assert preference[i][1] == worst restore_history(study_id, storage, history_id) - preference, history = get_preferences_history(history_id) + preference, history = get_preferences_history(study_id, storage, history_id) assert len(preference) == 2 From 702b35622e558b00d2ba42b546d8215d750553fc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 17:54:42 +0900 Subject: [PATCH 060/104] fix by review --- optuna_dashboard/_app.py | 2 +- optuna_dashboard/_preference_setting.py | 20 +++++++++++++------- python_tests/test_api.py | 6 +++--- python_tests/test_preference_setting.py | 4 ++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 2ad7874e..d7ddf72b 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -316,7 +316,7 @@ def create_app( except ValueError: response.status = 400 return {"reason": "invalid request."} - if component_type not in ["Note", "Artifact"]: + if component_type not in ["note", "artifact"]: response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d9d7c1c2..2310aa03 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,6 +1,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from typing import Any from optuna.storages import BaseStorage @@ -10,7 +11,7 @@ from .preferential._study import PreferentialStudy if TYPE_CHECKING: from typing import Literal - OUTPUT_COMPONENT_TYPE = Literal["Note", "Artifact"] + OUTPUT_COMPONENT_TYPE = Literal["note", "artifact"] _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" @@ -19,22 +20,22 @@ def _register_preference_feedback_component_type( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: + value: dict[str, Any] = {"type": component_type} + if artifact_key is not None: + value["artifact_key"] = artifact_key storage.set_study_system_attr( study_id=study_id, key=_SYSTEM_ATTR_FEEDBACK_COMPONENT, - value={ - "type": component_type, - "artifact_key": artifact_key, - }, + value=value, ) def register_preference_feedback_component_type( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, - artifact_key: str = "", + artifact_key: str | None = None, ) -> None: """Register output component to the study. @@ -48,6 +49,11 @@ def register_preference_feedback_component_type( this argument is used as the attribute key of the artifact. Each trial displays the artifact whose id is the value of the attribute. """ + if component_type == "artifact": + assert ( + artifact_key is not None + ), "artifact_key must be specified when component_type is Artifact" + _register_preference_feedback_component_type( study_id=study._study._study_id, storage=study._study._storage, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 1908c188..c478dc26 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -183,7 +183,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "Note") + register_preference_feedback_component_type(study, "note") for _ in range(3): study.ask() @@ -193,7 +193,7 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference_feedback_component_type", "PUT", - body=json.dumps({"type": "Artifact", "artifact_key": "image"}), + body=json.dumps({"type": "artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) @@ -207,7 +207,7 @@ class APITestCase(TestCase): self.assertEqual(status, 200) study_detail = json.loads(body) - assert study_detail["feedback_component_type"]["type"] == "Artifact" + assert study_detail["feedback_component_type"]["type"] == "artifact" assert study_detail["feedback_component_type"]["artifact_key"] == "image" def test_skip_trial(self) -> None: diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index 8a601af7..bcd53e42 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -11,10 +11,10 @@ from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "Artifact", "image_key") + register_preference_feedback_component_type(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type - assert feedback_type["type"] == "Artifact" + assert feedback_type["type"] == "artifact" assert "artifact_key" in feedback_type assert feedback_type["artifact_key"] == "image_key" From 53794eac73cf8029569811a17e254b6f28646bcc Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 18:05:51 +0900 Subject: [PATCH 061/104] fix by review --- optuna_dashboard/_app.py | 6 +++--- optuna_dashboard/_preferential_history.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index a020c8b8..0249de48 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,8 +28,8 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preferential_history import HistoryIdError from ._preferential_history import NewHistory +from ._preferential_history import PreferenceHistoryNotFound from ._preferential_history import remove_history from ._preferential_history import report_history from ._preferential_history import restore_history @@ -314,7 +314,7 @@ def create_app( def remove_preference(study_id: int, history_id: str) -> dict[str, Any]: try: remove_history(study_id, storage, history_id) - except HistoryIdError: + except PreferenceHistoryNotFound: response.status = 404 return {"reason": f"history_id={history_id} is not found"} @@ -326,7 +326,7 @@ def create_app( def restore_preference(study_id: int, history_id: str) -> dict[str, Any]: try: restore_history(study_id, storage, history_id) - except HistoryIdError: + except PreferenceHistoryNotFound: response.status = 404 return {"reason": f"history_id={history_id} is not found"} diff --git a/optuna_dashboard/_preferential_history.py b/optuna_dashboard/_preferential_history.py index 9726d27c..ef3f87f4 100644 --- a/optuna_dashboard/_preferential_history.py +++ b/optuna_dashboard/_preferential_history.py @@ -39,7 +39,7 @@ if TYPE_CHECKING: ) -class HistoryIdError(Exception): +class PreferenceHistoryNotFound(Exception): pass @@ -66,7 +66,7 @@ def report_history( else: assert False, f"Unknown data: {input_data}" - id = report_preferences( + preference_id = report_preferences( study_id=study_id, storage=storage, preferences=preferences, @@ -75,27 +75,27 @@ def report_history( if input_data.mode == "ChooseWorst": history: ChooseWorstHistory = { "mode": "ChooseWorst", - "id": id, + "id": preference_id, "timestamp": datetime.now().isoformat(), "candidates": input_data.candidates, "clicked": input_data.clicked, "preferences": preferences, } - key = _SYSTEM_ATTR_PREFIX_HISTORY + id + key = _SYSTEM_ATTR_PREFIX_HISTORY + preference_id storage.set_study_system_attr( study_id=study_id, key=key, value=json.dumps(history), ) - return id + return preference_id def remove_history(study_id: int, storage: BaseStorage, history_id: str) -> None: system_attrs = storage.get_study_system_attrs(study_id) history_key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id if history_key not in system_attrs: - raise HistoryIdError + raise PreferenceHistoryNotFound storage.set_study_system_attr(study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, []) @@ -103,7 +103,7 @@ def restore_history(study_id: int, storage: BaseStorage, history_id: str) -> Non system_attrs = storage.get_study_system_attrs(study_id) history_key = _SYSTEM_ATTR_PREFIX_HISTORY + history_id if history_key not in system_attrs: - raise HistoryIdError + raise PreferenceHistoryNotFound history: History = json.loads(system_attrs.get(history_key, "")) storage.set_study_system_attr( study_id, _SYSTEM_ATTR_PREFIX_PREFERENCE + history_id, history["preferences"] From 8e23d30d182f757def64b0c907d7d88a6517e45e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 12 Sep 2023 18:18:19 +0900 Subject: [PATCH 062/104] fix by format --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 2310aa03..acc64ccd 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -1,7 +1,7 @@ from __future__ import annotations -from typing import TYPE_CHECKING from typing import Any +from typing import TYPE_CHECKING from optuna.storages import BaseStorage From 32a4f2d4e1c267dba695c9e4828c376e32e5eb95 Mon Sep 17 00:00:00 2001 From: contramundum53 Date: Thu, 14 Sep 2023 10:36:20 +0900 Subject: [PATCH 063/104] Update test_api.py --- python_tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 2e46be43..d5de330d 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,8 +8,8 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history from optuna_dashboard._serializer import serialize_preference_history From 2422555475586f71a129ae4f5f6a88ad8c65a5e8 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 11:53:07 +0900 Subject: [PATCH 064/104] modify fail trial --- .../ts/components/PreferentialTrials.tsx | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index dc6e6c95..ecb2a7e2 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -192,16 +192,22 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ display: [], clicked: [], }) - const new_trails = activeTrials.filter( + const newTrials = activeTrials.filter( (t) => !displayTrials.display.includes(t.number) && !displayTrials.clicked.includes(t.number) ) - if (new_trails.length > 0) { + const deleteTrials = displayTrials.display.filter( + (t) => t !== -1 && !activeTrials.map((t) => t.number).includes(t) + ) + console.log(deleteTrials) + if (newTrials.length > 0 || deleteTrials.length > 0) { setDisplayTrials((prev) => { - const display = [...prev.display] + const display = [...prev.display].map((t) => + deleteTrials.includes(t) ? -1 : t + ) const clicked = [...prev.clicked] - new_trails.map((t) => { + newTrials.map((t) => { const index = display.findIndex((n) => n === -1) if (index === -1) { display.push(t.number) @@ -250,7 +256,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ {displayTrials.display.map((t, index) => ( trial.number === t)} candidates={displayTrials.display.filter((n) => n !== -1)} hideTrial={() => { From 88e3d511d5ff607d44b780205849b92efa6b6792 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 12:03:17 +0900 Subject: [PATCH 065/104] fix by review --- optuna_dashboard/_app.py | 4 +++- optuna_dashboard/_serializer.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 1baf76fe..4c22bdbf 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -312,7 +312,7 @@ def create_app( @app.put("/api/studies//preference_feedback_component_type") @json_api_view - def put_component(study_id: int) -> dict[str, Any]: + def put_preference_feedback_component_type(study_id: int) -> dict[str, Any]: try: component_type = request.json.get("type", "") artifact_key = request.json.get("artifact_key", None) @@ -329,6 +329,8 @@ def create_app( component_type=component_type, artifact_key=artifact_key, ) + response.status = 204 + return {} @app.delete("/api/studies//preference/") @json_api_view diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index ea22644f..b229b738 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -168,7 +168,6 @@ def serialize_study_detail( serialized["feedback_component_type"] = system_attrs.get( _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} ) - if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) serialized["plotly_graph_objects"] = [ From 4e031585b2d90b290d6bcd1dca5ccd50547a448a Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 14 Sep 2023 13:40:19 +0900 Subject: [PATCH 066/104] Support all-categorical cases --- optuna_dashboard/preferential/samplers/gp.py | 42 +++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 8349b1a4..349da03a 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -3,6 +3,7 @@ from __future__ import annotations import math from typing import Any from typing import Callable +from typing import cast import botorch.acquisition.analytic import botorch.models.model @@ -16,6 +17,8 @@ import optuna import optuna._transform import torch from torch import Tensor +from optuna.distributions import CategoricalDistribution +import itertools from .._system_attrs import get_preferences @@ -310,7 +313,7 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): search_space: dict[str, optuna.distributions.BaseDistribution], ) -> dict[str, Any]: preferences = get_preferences(study.system_attrs) - if len(preferences) == 0: + if len(preferences) == 0 or len(search_space) == 0: return {} trials = study.get_trials(deepcopy=False) @@ -355,16 +358,33 @@ class PreferentialGPSampler(optuna.samplers.BaseSampler): best_f=torch.max(sampled_gp.posterior(params[:, None, :]).mean), ) - # TODO: Make it possible to apply it on categorical variables - candidates, _ = botorch.optim.optimize_acqf( - acq_function=acqf, - bounds=torch.from_numpy(trans.bounds.T), - q=1, - num_restarts=10, - raw_samples=512, - options={"batch_limit": 5, "maxiter": 200}, - sequential=True, - ) + # TODO: Make it possible to apply it on mixed search space + if all(isinstance(dist, CategoricalDistribution) for dist in search_space.values()): + all_param_combinations = itertools.product( + *[ + [(name, choice) for choice in cast(CategoricalDistribution, dist).choices] + for name, dist in search_space.items() + ] + ) + choices = torch.tensor( + np.array([trans.transform(dict(params)) for params in all_param_combinations]), + dtype=torch.float64, + ) + candidates, _ = botorch.optim.optimize_acqf_discrete( + acq_function=acqf, + choices=choices, + q=1, + ) + else: + candidates, _ = botorch.optim.optimize_acqf( + acq_function=acqf, + bounds=torch.from_numpy(trans.bounds.T), + q=1, + num_restarts=10, + raw_samples=512, + options={"batch_limit": 5, "maxiter": 200}, + sequential=True, + ) next_x = trans.untransform(candidates[0].detach().numpy()) return next_x From b95f13e9fe9dc857bddf489666d8aea97374a8c1 Mon Sep 17 00:00:00 2001 From: Contramundum Date: Thu, 14 Sep 2023 13:45:22 +0900 Subject: [PATCH 067/104] format --- optuna_dashboard/preferential/samplers/gp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/optuna_dashboard/preferential/samplers/gp.py b/optuna_dashboard/preferential/samplers/gp.py index 349da03a..d023e741 100644 --- a/optuna_dashboard/preferential/samplers/gp.py +++ b/optuna_dashboard/preferential/samplers/gp.py @@ -1,5 +1,6 @@ from __future__ import annotations +import itertools import math from typing import Any from typing import Callable @@ -15,10 +16,9 @@ from gpytorch.likelihoods.gaussian_likelihood import Prior import numpy as np import optuna import optuna._transform +from optuna.distributions import CategoricalDistribution import torch from torch import Tensor -from optuna.distributions import CategoricalDistribution -import itertools from .._system_attrs import get_preferences From 3fdf22704c17dcb585629e0e6637104a8eda367e Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 14:33:25 +0900 Subject: [PATCH 068/104] fix feedback screen --- optuna_dashboard/_app.py | 6 ++++ optuna_dashboard/_serializer.py | 3 ++ optuna_dashboard/ts/apiClient.ts | 32 ++++++++++++------- .../ts/components/PreferentialTrials.tsx | 20 +++++++++--- optuna_dashboard/ts/types/index.d.ts | 3 ++ python_tests/test_serializers.py | 8 +++-- 6 files changed, 54 insertions(+), 18 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 0249de48..731de524 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -46,6 +46,7 @@ from .artifact._backend import register_artifact_route from .artifact._backend_to_store import to_artifact_store from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._study import get_best_trials as get_best_preferential_trials +from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import report_skip @@ -220,6 +221,10 @@ def create_app( ) = get_cached_extra_study_property(study_id, trials) plotly_graph_objects = get_plotly_graph_objects(system_attrs) + trials_id2number = {trial._trial_id: trial.number for trial in trials} + skipped_trials = [ + trials_id2number[trial_id] for trial_id in get_skipped_trial_ids(system_attrs) + ] return serialize_study_detail( summary, best_trials, @@ -229,6 +234,7 @@ def create_app( union_user_attrs, has_intermediate_values, plotly_graph_objects, + skipped_trials, ) @app.get("/api/studies//param_importances") diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 001f689a..9d2241be 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -19,6 +19,7 @@ from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import get_preferences +from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import is_preference_removed @@ -135,6 +136,7 @@ def serialize_study_detail( union_user_attrs: list[tuple[str, bool]], has_intermediate_values: bool, plotly_graph_objects: dict[str, str], + skipped_trials: list[int], ) -> dict[str, Any]: serialized: dict[str, Any] = { "name": summary.study_name, @@ -166,6 +168,7 @@ def serialize_study_detail( if serialized["is_preferential"]: serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) + serialized["skipped_trials"] = skipped_trials serialized["plotly_graph_objects"] = [ {"id": id_, "graph_object": graph_object} for id_, graph_object in plotly_graph_objects.items() diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index c0ad90f8..fefabeb4 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -56,24 +56,30 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } interface PreferenceHistoryResponce { - id: string - preference_id: string - candidates: number[] - clicked: number - mode: PreferenceFeedbackMode - timestamp: string + history: { + id: string + preference_id: string + candidates: number[] + clicked: number + mode: PreferenceFeedbackMode + timestamp: string + preferences: [number, number][] + } + is_removed: boolean } const convertPreferenceHistory = ( res: PreferenceHistoryResponce ): PreferenceHistory => { return { - id: res.id, - preference_id: res.preference_id, - candidates: res.candidates, - clicked: res.clicked, - feedback_mode: res.mode, - timestamp: new Date(res.timestamp), + id: res.history.id, + preference_id: res.history.preference_id, + candidates: res.history.candidates, + clicked: res.history.clicked, + feedback_mode: res.history.mode, + timestamp: new Date(res.history.timestamp), + preferences: res.history.preferences, + is_removed: res.is_removed, } } @@ -95,6 +101,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] + skipped_trials?: number[] } export const getStudyDetailAPI = ( @@ -135,6 +142,7 @@ export const getStudyDetailAPI = ( convertPreferenceHistory ), plotly_graph_objects: res.data.plotly_graph_objects, + skipped_trials: res.data.skipped_trials ?? [], } }) } diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index ecb2a7e2..6b0ea1f6 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -7,6 +7,7 @@ import { CardContent, CardActions, CardActionArea, + CircularProgress, } from "@mui/material" import ClearIcon from "@mui/icons-material/Clear" import IconButton from "@mui/material/IconButton" @@ -111,7 +112,11 @@ const PreferentialTrial: FC<{ padding: theme.spacing(2), }} > - + {trial.note.body !== "" ? ( + + ) : ( + + )} = ({ } const theme = useTheme() - const runningTrials = studyDetail.trials.filter((t) => t.state === "Running") - const activeTrials = runningTrials.concat(studyDetail.best_trials) + const hiddenTrials = new Set( + studyDetail.preference_history + ?.map((p) => p.clicked) + .concat(studyDetail.skipped_trials) ?? [] + ) + const activeTrials = studyDetail.trials.filter( + (t) => + (t.state === "Running" || t.state === "Complete") && + !hiddenTrials.has(t.number) + ) const [displayTrials, setDisplayTrials] = useState({ display: [], @@ -200,7 +213,6 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const deleteTrials = displayTrials.display.filter( (t) => t !== -1 && !activeTrials.map((t) => t.number).includes(t) ) - console.log(deleteTrials) if (newTrials.length > 0 || deleteTrials.length > 0) { setDisplayTrials((prev) => { const display = [...prev.display].map((t) => diff --git a/optuna_dashboard/ts/types/index.d.ts b/optuna_dashboard/ts/types/index.d.ts index 81096248..32f7ebd1 100644 --- a/optuna_dashboard/ts/types/index.d.ts +++ b/optuna_dashboard/ts/types/index.d.ts @@ -206,6 +206,7 @@ type StudyDetail = { preferences?: [number, number][] preference_history?: PreferenceHistory[] plotly_graph_objects: PlotlyGraphObject[] + skipped_trials: number[] } type StudyDetails = { @@ -222,4 +223,6 @@ type PreferenceHistory = { clicked: number feedback_mode: PreferenceFeedbackMode timestamp: Date + preferences: [number, number][] + is_removed: boolean } diff --git a/python_tests/test_serializers.py b/python_tests/test_serializers.py index 72db7b26..ea1c3517 100644 --- a/python_tests/test_serializers.py +++ b/python_tests/test_serializers.py @@ -29,7 +29,9 @@ def test_get_study_detail_is_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) + study_detail = serialize_study_detail( + study_summary, [], study.trials, [], [], [], False, {}, [] + ) assert study_detail["is_preferential"] @@ -40,7 +42,9 @@ def test_get_study_detail_is_not_preferential() -> None: assert len(study_summaries) == 1 study_summary = study_summaries[0] - study_detail = serialize_study_detail(study_summary, [], study.trials, [], [], [], False, {}) + study_detail = serialize_study_detail( + study_summary, [], study.trials, [], [], [], False, {}, [] + ) assert not study_detail["is_preferential"] From 1c3cb84d871e94b7441207aa350537545f8d0f53 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 14:40:52 +0900 Subject: [PATCH 069/104] fix by lint --- optuna_dashboard/_serializer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 9d2241be..d3cd786f 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -19,7 +19,6 @@ from ._preferential_history import _SYSTEM_ATTR_PREFIX_HISTORY from .artifact._backend import list_trial_artifacts from .preferential._study import _SYSTEM_ATTR_PREFERENTIAL_STUDY from .preferential._system_attrs import get_preferences -from .preferential._system_attrs import get_skipped_trial_ids from .preferential._system_attrs import is_preference_removed From 4d15b79e9b414a68e1f6d5f65e2828e54e35894f Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 15:20:46 +0900 Subject: [PATCH 070/104] fixed by merge --- optuna_dashboard/ts/components/PreferenceHistory.tsx | 6 +++--- optuna_dashboard/ts/components/PreferentialTrials.tsx | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 9e19bf07..db4df078 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -142,7 +142,7 @@ const ChoiceTrials: FC<{ trials: Trial[] study_id: number }> = ({ choice, trials, study_id }) => { - const [isRemoved, setRemoved] = useState(choice.isRemoved) + const [isRemoved, setRemoved] = useState(choice.is_removed) const theme = useTheme() const worst_trials = new Set([choice.clicked]) const action = actionCreator() @@ -170,7 +170,7 @@ const ChoiceTrials: FC<{ > {formatDate(choice.timestamp)} - {choice.isRemoved ? ( + {choice.is_removed ? ( { @@ -203,7 +203,7 @@ const ChoiceTrials: FC<{ display: "flex", flexDirection: "row", flexWrap: "wrap", - filter: choice.isRemoved ? "brightness(0.4)" : undefined, + filter: choice.is_removed ? "brightness(0.4)" : undefined, backgroundColor: theme.palette.background.paper, }} > diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 7fcb634f..26cc4976 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -196,7 +196,8 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ const hiddenTrials = new Set( studyDetail.preference_history - ?.map((p) => p.clicked) + ?.filter((h) => !h.is_removed) + .map((p) => p.clicked) .concat(studyDetail.skipped_trials) ?? [] ) const activeTrials = studyDetail.trials.filter( @@ -256,7 +257,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ }) } const latestHistoryId = - studyDetail?.preference_history?.filter((h) => !h.isRemoved).pop()?.id ?? + studyDetail?.preference_history?.filter((h) => !h.is_removed).pop()?.id ?? null if (undoHistoryId !== null && undoHistoryId !== latestHistoryId) { setUndoHistoryId(null) From e5e6d0766f69c69332d3c9d149e71f8638ade1c1 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 15:37:48 +0900 Subject: [PATCH 071/104] fix docstring --- optuna_dashboard/_preference_setting.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index acc64ccd..167fd178 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -37,17 +37,22 @@ def register_preference_feedback_component_type( component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, ) -> None: - """Register output component to the study. + """Register a preference feedback component to the study. + With this feature, you can change the component, displayed on the + human feedback pages. By default, the Markdown note (``component_type="note"``) + is displayed. If you specify ``component_type="artifact"``, the viewer for the + specified artifact file will be displayed. Args: study: - The study to register the output component. + The study to register the preference feedback component. component_type: - The type of the output component. - artifact_key: - When the component_type is "Artifact", - this argument is used as the attribute key of the artifact. - Each trial displays the artifact whose id is the value of the attribute. + The component type, displayed on the human feedback pages + (default: ``"note"``). + user_attr_artifact_key: + This option is required when the ``component_type`` is ``"artifact"``. + The user attribute, which is specified this field, must contain the + ``artifact``id you want to display on the human feedback page. """ if component_type == "artifact": assert ( From 9d7f2f9a6e141450f388747782ef9de2f228b141 Mon Sep 17 00:00:00 2001 From: moririn2528 <49509238+moririn2528@users.noreply.github.com> Date: Thu, 14 Sep 2023 16:28:43 +0900 Subject: [PATCH 072/104] Update optuna_dashboard/_preference_setting.py Co-authored-by: c-bata --- optuna_dashboard/_preference_setting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 167fd178..d0f8c82c 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -32,7 +32,7 @@ def _register_preference_feedback_component_type( ) -def register_preference_feedback_component_type( +def register_preference_feedback_component( study: PreferentialStudy, component_type: OUTPUT_COMPONENT_TYPE, artifact_key: str | None = None, From 4570efe74db693276482aa946bfec4be7cec3064 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:39:44 +0900 Subject: [PATCH 073/104] fix by review --- docs/api.rst | 1 + optuna_dashboard/_app.py | 4 ++-- optuna_dashboard/_preference_setting.py | 4 ++-- python_tests/test_api.py | 4 ++-- python_tests/test_preference_setting.py | 4 ++-- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/docs/api.rst b/docs/api.rst index aadd2718..e66ae468 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,6 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy + optuna_dashboard._prefential_setting.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index 4c22bdbf..3abac3fc 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -28,7 +28,7 @@ from ._cached_extra_study_property import get_cached_extra_study_property from ._custom_plot_data import get_plotly_graph_objects from ._importance import get_param_importance_from_trials_cache from ._pareto_front import get_pareto_front_trials -from ._preference_setting import _register_preference_feedback_component_type +from ._preference_setting import _register_preference_feedback_component from ._preferential_history import NewHistory from ._preferential_history import PreferenceHistoryNotFound from ._preferential_history import remove_history @@ -323,7 +323,7 @@ def create_app( response.status = 400 return {"reason": "component_type must be either 'Note' or 'Artifact'."} - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study_id, storage=storage, component_type=component_type, diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index d0f8c82c..907fc2be 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -16,7 +16,7 @@ if TYPE_CHECKING: _SYSTEM_ATTR_FEEDBACK_COMPONENT = "preference:component" -def _register_preference_feedback_component_type( +def _register_preference_feedback_component( study_id: int, storage: BaseStorage, component_type: OUTPUT_COMPONENT_TYPE, @@ -59,7 +59,7 @@ def register_preference_feedback_component( artifact_key is not None ), "artifact_key must be specified when component_type is Artifact" - _register_preference_feedback_component_type( + _register_preference_feedback_component( study_id=study._study._study_id, storage=study._study._storage, component_type=component_type, diff --git a/python_tests/test_api.py b/python_tests/test_api.py index d5de330d..db79c0d8 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -8,7 +8,7 @@ from optuna import get_all_study_summaries from optuna.study import StudyDirection from optuna_dashboard._app import create_app from optuna_dashboard._app import create_new_study -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard._preferential_history import NewHistory from optuna_dashboard._preferential_history import remove_history from optuna_dashboard._preferential_history import report_history @@ -187,7 +187,7 @@ class APITestCase(TestCase): def test_change_component(self) -> None: storage = optuna.storages.InMemoryStorage() study = create_study(storage=storage, n_generate=3) - register_preference_feedback_component_type(study, "note") + register_preference_feedback_component(study, "note") for _ in range(3): study.ask() diff --git a/python_tests/test_preference_setting.py b/python_tests/test_preference_setting.py index bcd53e42..a0adcb39 100644 --- a/python_tests/test_preference_setting.py +++ b/python_tests/test_preference_setting.py @@ -4,14 +4,14 @@ from unittest import TestCase import optuna from optuna_dashboard._preference_setting import _SYSTEM_ATTR_FEEDBACK_COMPONENT -from optuna_dashboard._preference_setting import register_preference_feedback_component_type +from optuna_dashboard._preference_setting import register_preference_feedback_component from optuna_dashboard.preferential._study import PreferentialStudy class FeedbackSettingTestCase(TestCase): def test_widget_to_dict_from_dict(self) -> None: study = PreferentialStudy(optuna.create_study()) - register_preference_feedback_component_type(study, "artifact", "image_key") + register_preference_feedback_component(study, "artifact", "image_key") system_attrs = study._study.system_attrs feedback_type = system_attrs.get(_SYSTEM_ATTR_FEEDBACK_COMPONENT, {}) assert "type" in feedback_type From df1948a484f896aac4ef9e3576c36c7481a6ac13 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 16:42:58 +0900 Subject: [PATCH 074/104] minor fix --- docs/api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index e66ae468..8da91536 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._prefential_setting.register_preference_feedback_component + optuna_dashboard._preference_setting.register_preference_feedback_component Streamlit ----------------- From 9f17603741a73770ebebec2d87356d2d1ef7ccda Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 17:05:12 +0900 Subject: [PATCH 075/104] minor fix --- docs/api.rst | 2 +- optuna_dashboard/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/api.rst b/docs/api.rst index 8da91536..18f8bc72 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -44,7 +44,7 @@ Preferential Optimization optuna_dashboard.preferential.create_study optuna_dashboard.preferential.load_study optuna_dashboard.preferential.PreferentialStudy - optuna_dashboard._preference_setting.register_preference_feedback_component + optuna_dashboard.register_preference_feedback_component Streamlit ----------------- diff --git a/optuna_dashboard/__init__.py b/optuna_dashboard/__init__.py index 5bb3f301..ea2a8dbe 100644 --- a/optuna_dashboard/__init__.py +++ b/optuna_dashboard/__init__.py @@ -14,6 +14,7 @@ from ._form_widget import TextInputWidget # noqa from ._named_objectives import set_objective_names # noqa from ._note import get_note # noqa from ._note import save_note # noqa +from ._preference_setting import register_preference_feedback_component # noqa __version__ = "0.13.0b1" From eb6234eabc683e82c9c09379d938cdb400a631c1 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 17:39:06 +0900 Subject: [PATCH 076/104] erase isolated node from graph --- optuna_dashboard/ts/components/PreferentialGraph.tsx | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 48364598..f343cb06 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -181,6 +181,7 @@ export const PreferentialGraph: FC<{ if (!studyDetail.is_preferential || studyDetail.preferences === undefined) return const preferences = reductionPreference(studyDetail.preferences) + const trialNodes = Array.from(new Set(preferences.flat())) const graph: ElkNode = { id: "root", layoutOptions: { @@ -189,8 +190,8 @@ export const PreferentialGraph: FC<{ "elk.layered.spacing.nodeNodeBetweenLayers": nodeMargin.toString(), "elk.spacing.nodeNode": nodeMargin.toString(), }, - children: studyDetail.trials.map((trial) => ({ - id: `${trial.number}`, + children: trialNodes.map((trial) => ({ + id: `${trial}`, targetPosition: "top", sourcePosition: "bottom", width: nodeWidth, @@ -207,7 +208,7 @@ export const PreferentialGraph: FC<{ .then((layoutedGraph) => { setNodes( layoutedGraph.children?.map((node, index) => { - const trial = studyDetail.trials[index] + const trial = studyDetail.trials[trialNodes[index]] return { id: `${trial.number}`, type: "note", From 098a607de492a552dadcae67ca80b38c912c3de9 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 14 Sep 2023 19:11:03 +0900 Subject: [PATCH 077/104] fix by changed api, reloading --- optuna_dashboard/_app.py | 2 +- optuna_dashboard/_serializer.py | 4 - optuna_dashboard/ts/apiClient.ts | 19 ++- optuna_dashboard/ts/components/AppDrawer.tsx | 4 +- .../ts/components/PreferentialTrials.tsx | 154 ++++++++++++------ optuna_dashboard/ts/types/index.d.ts | 2 +- 6 files changed, 115 insertions(+), 70 deletions(-) diff --git a/optuna_dashboard/_app.py b/optuna_dashboard/_app.py index abd717d0..5b1f7147 100644 --- a/optuna_dashboard/_app.py +++ b/optuna_dashboard/_app.py @@ -327,7 +327,7 @@ def create_app( return {"reason": "invalid request."} if component_type not in ["note", "artifact"]: response.status = 400 - return {"reason": "component_type must be either 'Note' or 'Artifact'."} + return {"reason": "component_type must be either 'note' or 'artifact'."} _register_preference_feedback_component( study_id=study_id, diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 8e3ffee2..5829f35d 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,10 +165,6 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets - if _SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE in system_attrs: - serialized["feedback_component_type"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_COMPONENT_TYPE] - if _SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY in system_attrs: - serialized["feedback_artifact_key"] = system_attrs[_SYSTEM_ATTR_FEEDBACK_ARTIFACT_KEY] if serialized["is_preferential"]: serialized["feedback_component_type"] = system_attrs.get( _SYSTEM_ATTR_FEEDBACK_COMPONENT, {} diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index 30eec71d..d2a786cf 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -83,6 +83,10 @@ const convertPreferenceHistory = ( } } +interface FeedbackComponentResponse { + type: string + artifact_key?: string +} interface StudyDetailResponse { name: string datetime_start: string @@ -101,8 +105,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] - feedback_component_type?: FeedbackComponentType - feedback_artifact_key?: string + feedback_component_type?: FeedbackComponentResponse skipped_trials?: number[] } @@ -139,10 +142,10 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data.feedback_component_type - ? (res.data.feedback_component_type as FeedbackComponentType) - : "Note", - feedback_artifact_key: res.data.feedback_artifact_key, + feedback_component_type: res.data.feedback_component_type?.type + ? (res.data.feedback_component_type.type as FeedbackComponentType) + : "note", + feedback_artifact_key: res.data.feedback_component_type?.artifact_key, preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory @@ -389,8 +392,8 @@ export const reportFeedbackComponentAPI = ( artifact_key?: string ): Promise => { return axiosInstance - .post(`/api/studies/${studyId}/component`, { - component_type: component_type, + .put(`/api/studies/${studyId}/preference_feedback_component_type`, { + type: component_type, artifact_key: artifact_key, }) .then(() => { diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index 8740c68a..b9117707 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -287,7 +287,7 @@ export const AppDrawer: FC<{ - + - + diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 36cf9ab9..26ede44d 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useEffect, useState } from "react" +import React, { FC, useEffect, useState, useMemo } from "react" import { Typography, Box, @@ -76,7 +76,7 @@ const SettingsPage: FC<{ const theme = useTheme() const actions = actionCreator() const [outputComponent, setOutputComponent] = useState( - studyDetail?.feedback_component_type ?? "Note" + studyDetail?.feedback_component_type ?? "note" ) const [outputArtifactKey, setOutputArtifactKey] = useState( studyDetail?.feedback_artifact_key ?? "" @@ -124,11 +124,11 @@ const SettingsPage: FC<{ setOutputComponent(e.target.value as FeedbackComponentType) }} > - Note - Artifact + Note + Artifact - {outputComponent === "Artifact" ? ( + {outputComponent === "artifact" ? ( { + if (componentId === undefined || componentId === "note") { + return trial.note.body !== "" + } + if (componentId === "artifact") { + const artifactId = trial?.user_attrs.find( + (a) => a.key === artifactKey + )?.value + const artifact = trial?.artifacts.find((a) => a.artifact_id === artifactId) + return artifact !== undefined + } + return false +} + export const OutputContent: FC<{ trial: Trial artifact?: Artifact componentId?: FeedbackComponentType urlPath: string }> = ({ trial, artifact, componentId, urlPath }) => { - if ( - (componentId === undefined || componentId === "Note") && - trial.note.body !== "" - ) { + const note = useMemo(() => { return + }, [trial.note.body]) + if (componentId === undefined || componentId === "note") { + return note } - if (componentId === "Artifact" && artifact !== undefined) { + if (componentId === "artifact") { + if (artifact === undefined) { + return null + } return ( ) } - - return + return null } export const getArtifactUrlPath = ( @@ -222,7 +242,7 @@ const PreferentialTrial: FC<{ ? getArtifactUrlPath(studyDetail.id, trial?.trial_id, artifactId) : "" const is3dModel = - componentId === "Artifact" && + componentId === "artifact" && artifact !== undefined && isThreejsArtifact(artifact) @@ -242,6 +262,7 @@ const PreferentialTrial: FC<{ hideTrial() action.updatePreference(trial.study_id, candidates, trial.number) } + const isReady = isComparisonReady(trial, componentId, artifactKey) return ( Trial {trial.number} - {componentId === "Artifact" && artifact !== undefined ? ( + {componentId === "artifact" && artifact !== undefined ? ( - - - + {isReady ? ( + <> + + + + + ) : ( + + )} + + + ) } @@ -571,22 +550,51 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ studyDetail={studyDetail} /> {detailTrial !== null && ( - { - setDetailTrial(null) - }} - > - - studyDetail.trials.find((t) => t.trial_id === trialId)?.state === - "Complete" ?? false - } - directions={[]} - objectiveNames={[]} - /> - + setDetailTrial(null)}> + + + + + + + studyDetail.trials.find((t) => t.trial_id === trialId) + ?.state === "Complete" ?? false + } + directions={[]} + objectiveNames={[]} + /> + + + )} {renderThreejsArtifactModal()} From 9a4f91ce07962cd7683c8867ef974f372bde7dfa Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 15 Sep 2023 19:30:58 +0900 Subject: [PATCH 090/104] split output component --- .../ts/components/PreferenceHistory.tsx | 5 ++-- .../ts/components/PreferentialGraph.tsx | 5 ++-- .../PreferentialOutputComponent.tsx | 26 ++++++++++++++++ .../ts/components/PreferentialTrials.tsx | 30 ++----------------- 4 files changed, 35 insertions(+), 31 deletions(-) create mode 100644 optuna_dashboard/ts/components/PreferentialOutputComponent.tsx diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index 97a19dfe..e9c4948c 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -14,9 +14,10 @@ import Modal from "@mui/material/Modal" import { red } from "@mui/material/colors" import { TrialListDetail } from "./TrialList" -import { OutputContent, getArtifactUrlPath } from "./PreferentialTrials" +import { getArtifactUrlPath } from "./PreferentialTrials" import { formatDate } from "../dateUtil" import { useStudyDetailValue } from "../state" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" type TrialType = "worst" | "none" @@ -93,7 +94,7 @@ const CandidateTrial: FC<{ padding: theme.spacing(2), }} > - > = ({ data, isConnectable }) => { isConnectable={isConnectable} /> - = ({ trial, artifact, componentType, urlPath }) => { + const note = useMemo(() => { + return + }, [trial.note.body]) + if (componentType === undefined || componentType.output_type === "note") { + return note + } + if (componentType.output_type === "artifact") { + if (artifact === undefined) { + return null + } + return ( + + ) + } + return null +} diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 120c317b..27f3a8d5 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -1,4 +1,4 @@ -import React, { FC, useEffect, useState, useMemo } from "react" +import React, { FC, useEffect, useState } from "react" import { Typography, Box, @@ -32,8 +32,7 @@ import { isThreejsArtifact, useThreejsArtifactModal, } from "./ThreejsArtifactViewer" -import { ArtifactCardMedia } from "./ArtifactCardMedia" -import { MarkdownRenderer } from "./Note" +import { PreferentialOutputComponent } from "./PreferentialOutputComponent" const SettingsPage: FC<{ studyDetail: StudyDetail @@ -169,29 +168,6 @@ const isComparisonReady = ( return false } -export const OutputContent: FC<{ - trial: Trial - artifact?: Artifact - componentType: FeedbackComponentType - urlPath: string -}> = ({ trial, artifact, componentType, urlPath }) => { - const note = useMemo(() => { - return - }, [trial.note.body]) - if (componentType === undefined || componentType.output_type === "note") { - return note - } - if (componentType.output_type === "artifact") { - if (artifact === undefined) { - return null - } - return ( - - ) - } - return null -} - export const getArtifactUrlPath = ( studyId: number, trialId: number, @@ -338,7 +314,7 @@ const PreferentialTrial: FC<{ > {isReady ? ( <> - Date: Tue, 19 Sep 2023 10:36:50 +0900 Subject: [PATCH 091/104] fix test --- python_tests/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python_tests/test_api.py b/python_tests/test_api.py index 86f897c8..f9c56e4f 100644 --- a/python_tests/test_api.py +++ b/python_tests/test_api.py @@ -197,7 +197,7 @@ class APITestCase(TestCase): app, f"/api/studies/{study_id}/preference_feedback_component", "PUT", - body=json.dumps({"type": "artifact", "artifact_key": "image"}), + body=json.dumps({"output_type": "artifact", "artifact_key": "image"}), content_type="application/json", ) self.assertEqual(status, 204) From e40f410264b35c5fc4f343309fdbbeabb53529d6 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Tue, 19 Sep 2023 13:38:42 +0900 Subject: [PATCH 092/104] fix by review --- optuna_dashboard/_serializer.py | 12 ++++++------ optuna_dashboard/ts/apiClient.ts | 6 ++---- optuna_dashboard/ts/components/AppDrawer.tsx | 4 ++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/optuna_dashboard/_serializer.py b/optuna_dashboard/_serializer.py index 016b7fb8..e1840df9 100644 --- a/optuna_dashboard/_serializer.py +++ b/optuna_dashboard/_serializer.py @@ -165,13 +165,13 @@ def serialize_study_detail( form_widgets = get_form_widgets_json(system_attrs) if form_widgets: serialized["form_widgets"] = form_widgets + serialized["feedback_component_type"] = system_attrs.get( + _SYSTEM_ATTR_FEEDBACK_COMPONENT, + { + "output_type": "note", + }, + ) if serialized["is_preferential"]: - serialized["feedback_component_type"] = system_attrs.get( - _SYSTEM_ATTR_FEEDBACK_COMPONENT, - { - "output_type": "note", - }, - ) serialized["preference_history"] = serialize_preference_history(system_attrs) serialized["preferences"] = get_preferences(system_attrs) serialized["skipped_trials"] = skipped_trials diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index e552f773..60c461db 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -101,7 +101,7 @@ interface StudyDetailResponse { preferences?: [number, number][] preference_history?: PreferenceHistoryResponce[] plotly_graph_objects: PlotlyGraphObject[] - feedback_component_type?: FeedbackComponentType + feedback_component_type: FeedbackComponentType skipped_trials?: number[] } @@ -138,9 +138,7 @@ export const getStudyDetailAPI = ( objective_names: res.data.objective_names, form_widgets: res.data.form_widgets, is_preferential: res.data.is_preferential, - feedback_component_type: res.data.feedback_component_type ?? { - output_type: "note", - }, + feedback_component_type: res.data.feedback_component_type, preferences: res.data.preferences, preference_history: res.data.preference_history?.map( convertPreferenceHistory diff --git a/optuna_dashboard/ts/components/AppDrawer.tsx b/optuna_dashboard/ts/components/AppDrawer.tsx index b9117707..8740c68a 100644 --- a/optuna_dashboard/ts/components/AppDrawer.tsx +++ b/optuna_dashboard/ts/components/AppDrawer.tsx @@ -287,7 +287,7 @@ export const AppDrawer: FC<{ - + - + From 41081175f374f946667fd87fe4e93d1601f004d1 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Wed, 20 Sep 2023 18:34:16 +0900 Subject: [PATCH 093/104] fix undo in feedback screen --- .../ts/components/PreferentialTrials.tsx | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 64429451..1e05eaf0 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -401,7 +401,7 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ }) => { const theme = useTheme() const action = actionCreator() - const [undoHistoryId, setUndoHistoryId] = useState(null) + const [undoHistoryFlag, setUndoHistoryFlag] = useState(false) const [openThreejsArtifactModal, renderThreejsArtifactModal] = useThreejsArtifactModal() const [displayTrials, setDisplayTrials] = useState({ @@ -472,12 +472,23 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ } }) } + const visibleTrial = (num: number) => { + setDisplayTrials((prev) => { + const index = prev.clicked.findIndex((n) => n === num) + if (index === -1) { + return prev + } + const clicked = [...prev.clicked] + clicked[index] = -1 + return { + display: prev.display, + clicked: clicked, + } + }) + } const latestHistoryId = studyDetail?.preference_history?.filter((h) => !h.is_removed).pop()?.id ?? null - if (undoHistoryId !== null && undoHistoryId !== latestHistoryId) { - setUndoHistoryId(null) - } return ( @@ -500,13 +511,18 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ > From 1dd384933d0b131db68ef5a629874ad84e798f1b Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Thu, 21 Sep 2023 11:41:59 +0900 Subject: [PATCH 096/104] minor fix --- optuna_dashboard/ts/components/PreferentialTrials.tsx | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/optuna_dashboard/ts/components/PreferentialTrials.tsx b/optuna_dashboard/ts/components/PreferentialTrials.tsx index 85140011..b3934a80 100644 --- a/optuna_dashboard/ts/components/PreferentialTrials.tsx +++ b/optuna_dashboard/ts/components/PreferentialTrials.tsx @@ -510,6 +510,10 @@ export const PreferentialTrials: FC<{ studyDetail: StudyDetail | null }> = ({ From 2967e42339945f9b42b5945679e991335f686a30 Mon Sep 17 00:00:00 2001 From: c-bata Date: Fri, 22 Sep 2023 11:49:08 +0900 Subject: [PATCH 097/104] Update preferential optimization tutorial --- docs/tutorials/preferential-optimization.rst | 71 +++++++++---------- .../preferential-optimization/generator.py | 25 +++---- optuna_dashboard/_preference_setting.py | 1 + 3 files changed, 42 insertions(+), 55 deletions(-) diff --git a/docs/tutorials/preferential-optimization.rst b/docs/tutorials/preferential-optimization.rst index 706eabe8..4d2644ea 100644 --- a/docs/tutorials/preferential-optimization.rst +++ b/docs/tutorials/preferential-optimization.rst @@ -4,67 +4,66 @@ Tutorial: Preferential Optimization What is Preferential Optimization? ---------------------------------- -Preferential optimization is the way to optimize hyperparameters based on human preferences, -specifically by determining which trial is better when given a pair to compare. -Compared to the `human-in-the-loop optimization utilizing objective form widgets `_, -which relies on absolute evaluations, preferential optimization significantly reduces fluctuations in the evaluators' criteria, -ensuring more consistent results. +Preferential optimization is a method for optimizing hyperparameters, focusing of human preferences, by determining which trial is superior when comparing a pair. +It differs from `human-in-the-loop optimization utilizing objective form widgets `_, +which relies on absolute evaluations, as it significantly reduces fluctuations in evaluators' criteria, thus ensuring more consistent results. -In this tutorial, we will interactively optimize RGB values between 0 and 255 to generate a color that resembles the "sunset hue", which is the same problem setting as `this tutorial `_. -Hence, familiarizing yourself with the tutorial on objective form widgets beforehand might offer a smoother understanding. +In this tutorial, we'll interactively optimize RGB values to generate a color resembling a "sunset hue", +aligining with the problem setting in `this tutorial `_. +Familiarity with the tutorial ob objective form widgets may enhance your understanding. How to Run Preferential Optimization ------------------------------------ -In preferential optimization, we run two programs simultaneously: `generator.py`_ which executes parameter sampling or image generation, -and the Optuna Dashboard which provides a user interface for human evaluation. +In preferential optimization, two programs run concurrently: `generator.py`_ performing parameter sampling and image generation, +and the Optuna Dashboard, offering a user interface for human evaluation. .. figure:: ./images/preferential-optimization/system-architecture.png :alt: System Architecture :align: center :width: 800px -To start, ensure you have the necessary packages installed. You can do this by running the following command in your terminal: +First, ensure the necessary packages are installed by executing the following command in your terminal: .. code-block:: console $ pip install "optuna>=3.3.0" "optuna-dashboard>=0.13.0b1" pillow botorch -Run a Python script below which you copied from `generator.py`_. +Next, execute the Python script, copied from `generator.py`_. .. code-block:: console $ python generator.py -Then run a following command to launch Optuna Dashboard in a separate process. +Then, launch Optuna Dashboard in a separate process using the following command. .. code-block:: console $ optuna-dashboard sqlite:///example.db --artifact-dir ./artifact -In the command, the storage is set to ``sqlite:///example.db`` to persist Optuna's trial history. -To store the artifacts (output images), ``--artifact-dir ./artifact`` is specified. +Here, the storage is configured to ``sqlite:///example.db`` to retain Optuna's trial history, +and ``--artifact-dir ./artifact`` is specified to store the artifacts (output images). .. code-block:: console Listening on http://127.0.0.1:8080/ Hit Ctrl-C to quit. -When you run the command, you will see a message like the one above. -Please open `http://127.0.0.1:8080/dashboard/ `_ in your browser, then you can see the Optuna Dashboard as follows: +Upon executing the command, a message like the above will appear. +Open `http://127.0.0.1:8080/dashboard/ `_ in your browser to view the Optuna Dashboard: .. figure:: ./images/preferential-optimization/anim.gif :alt: GIF animation for preferential optimization :align: center :width: 800px - Selecting the least sunset-like color from four trials to report human preferences. + Select the least sunset-like color from four trials to record human preferences. Script Explanation ------------------ -Here, we specify the SQLite database URL and setup the artifact store, a filesystem to store images generated during the trial. +First, we specify the SQLite database URL and initialize the artifact store to house the images produced during the trial. .. code-block:: python :linenos: @@ -74,29 +73,35 @@ Here, we specify the SQLite database URL and setup the artifact store, a filesys artifact_store = FileSystemArtifactStore(base_path=artifact_path) os.makedirs(artifact_path, exist_ok=True) -Within the ``main()`` function, we initialize the study with necessary parameters, including specifying the preferential sampler. -ote that the ``Study`` and ``Sampler`` instantiated here are different from the conventional Optuna's ``Study``` and the ``Sampler``. -Preferential optimization relies solely on the comparison results between trials, and there are no absolute evaluation values for each trial. -Therefore, it is necessary to create dedicated ``Study`` and ``Sampler`` objects. +Within the ``main()`` function, creating dedicated ``Study`` and ``Sampler`` objects since preferential optimization relies on the comparison results between trials, lacking absolute evaluation values for each one. + +Then, the component to be displayed on the human feedback pages is registered via :func:`~optuna_dashboard.register_preference_feedback_component`. +The generated images are uploaded to the artifact store, and their ``artifact_id`` is stored in the trial user attribute (e.g., ``trial.user_attrs["rgb_image"]``), +enabling the Optuna Dashboard to display images on the evaluation feedback page. .. code-block:: python :linenos: + from optuna_dashboard import register_preference_feedback_component from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler study = create_study( - n_generate=5, + n_generate=4, study_name="Preferential Optimization", storage=STORAGE_URL, sampler=PreferentialGPSampler(), load_if_exists=True, ) + # Change the component, displayed on the human feedback pages. + # By default (component_type="note"), the Trial's Markdown note is displayed. + user_attr_key = "rgb_image" + register_preference_feedback_component(study, "artifact", user_attr_key) -Then, we create a loop that continuously checks if new trials should be generated, awaiting human evaluation if not. +Following this, we create a loop that continuously checks if new trials should be generated, awaiting human evaluation if not. Within the while loop, new trials are generated if the condition :meth:`~optuna_dashboard.preferential.PreferentialStudy.should_generate` returns ``True``. -For each trial, RGB values are sampled, and an image is generated with these values. -The image is saved temporarily, uploaded to artifact store, and then saved a Markdown note using :func:`~optuna_dashboard.save_note`. +For each trial, RGB values are sampled, an image is generated with these values, saved temporarily. +Then the image is uploaded to the artifact store, and finally, the ``artifact_id`` is stored to the key, which is specified via :func:`~optuna_dashboard.register_preference_feedback_component`. .. code-block:: python :linenos: @@ -118,18 +123,8 @@ The image is saved temporarily, uploaded to artifact store, and then saved a Mar image = Image.new("RGB", (320, 240), color=(r, g, b)) image.save(image_path) - # Upload to Artifact store + # Upload Artifact and set artifact_id to trial.user_attrs["rgb_image"]. artifact_id = upload_artifact(trial, image_path, artifact_store) - trial.set_user_attr("artifact_id", artifact_id) - print("RGB:", (r, g, b)) - - # Save a Markdown note - note = textwrap.dedent( - f"""\ - ![generated-image]({get_artifact_path(trial, artifact_id)}) - - (R, G, B) = ({r}, {g}, {b}) - """ - ) + trial.set_user_attr(user_attr_key, artifact_id) .. _generator.py: https://github.com/optuna/optuna-dashboard/blob/main/examples/preferential-optimization/generator.py diff --git a/examples/preferential-optimization/generator.py b/examples/preferential-optimization/generator.py index 66be1765..a26343bb 100644 --- a/examples/preferential-optimization/generator.py +++ b/examples/preferential-optimization/generator.py @@ -2,14 +2,12 @@ from __future__ import annotations import os import tempfile -import textwrap import time from typing import NoReturn from optuna.artifacts import FileSystemArtifactStore from optuna.artifacts import upload_artifact -from optuna_dashboard import save_note -from optuna_dashboard.artifact import get_artifact_path +from optuna_dashboard import register_preference_feedback_component from optuna_dashboard.preferential import create_study from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler from PIL import Image @@ -23,12 +21,16 @@ os.makedirs(artifact_path, exist_ok=True) def main() -> NoReturn: study = create_study( - n_generate=5, + n_generate=4, study_name="Preferential Optimization", storage=STORAGE_URL, sampler=PreferentialGPSampler(), load_if_exists=True, ) + # Change the component, displayed on the human feedback pages. + # By default (component_type="note"), the Trial's Markdown note is displayed. + user_attr_key = "rgb_image" + register_preference_feedback_component(study, "artifact", user_attr_key) with tempfile.TemporaryDirectory() as tmpdir: while True: @@ -49,20 +51,9 @@ def main() -> NoReturn: image = Image.new("RGB", (320, 240), color=(r, g, b)) image.save(image_path) - # 3. Upload Artifact + # 3. Upload Artifact and set artifact_id to trial.user_attrs["rgb_image"]. artifact_id = upload_artifact(trial, image_path, artifact_store) - trial.set_user_attr("artifact_id", artifact_id) - print("RGB:", (r, g, b)) - - # 4. Save Note - note = textwrap.dedent( - f"""\ - ![generated-image]({get_artifact_path(trial, artifact_id)}) - - (R, G, B) = ({r}, {g}, {b}) - """ - ) - save_note(trial, note) + trial.set_user_attr(user_attr_key, artifact_id) if __name__ == "__main__": diff --git a/optuna_dashboard/_preference_setting.py b/optuna_dashboard/_preference_setting.py index 907fc2be..a1fb217e 100644 --- a/optuna_dashboard/_preference_setting.py +++ b/optuna_dashboard/_preference_setting.py @@ -43,6 +43,7 @@ def register_preference_feedback_component( human feedback pages. By default, the Markdown note (``component_type="note"``) is displayed. If you specify ``component_type="artifact"``, the viewer for the specified artifact file will be displayed. + Args: study: The study to register the preference feedback component. From 0903eb54eefecfc28dd3a84b678c08e0c5401880 Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Fri, 22 Sep 2023 13:57:11 +0900 Subject: [PATCH 098/104] Add tests for samplers of preferential optimization --- python_tests/preferential/test_samplers.py | 117 +++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 python_tests/preferential/test_samplers.py diff --git a/python_tests/preferential/test_samplers.py b/python_tests/preferential/test_samplers.py new file mode 100644 index 00000000..00b74ded --- /dev/null +++ b/python_tests/preferential/test_samplers.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from collections.abc import Callable + +import optuna +from optuna import create_trial +from optuna.distributions import CategoricalDistribution +from optuna.distributions import FloatDistribution +from optuna.distributions import IntDistribution +from optuna.samplers import BaseSampler +from optuna.trial import TrialState +from optuna_dashboard.preferential import create_study +from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler +import pytest + + +parametrize_sampler = pytest.mark.parametrize( + "sampler_class", [optuna.samplers.RandomSampler, PreferentialGPSampler] +) + + +@parametrize_sampler +def test_sample_float(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + + for i in range(5): + past_trial = create_trial( + state=TrialState.RUNNING, + params={"x": 1.0}, + distributions={"x": FloatDistribution(0, 10)}, + ) + study.add_trial(past_trial) + study.report_preference(study.trials[:-1], study.trials[-1]) + + trial = study.ask() + trial.suggest_float("x", 0, 10) + + +@parametrize_sampler +def test_sample_int(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + + for i in range(5): + past_trial = create_trial( + state=TrialState.RUNNING, + params={"x": 1}, + distributions={"x": IntDistribution(0, 10)}, + ) + study.add_trial(past_trial) + study.report_preference(study.trials[:-1], study.trials[-1]) + + trial = study.ask() + trial.suggest_int("x", 0, 10) + + +@parametrize_sampler +def test_sample_categorical(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + + for i in range(5): + past_trial = create_trial( + state=TrialState.RUNNING, + params={"x": "A"}, + distributions={"x": CategoricalDistribution(["A", "B", "C"])}, + ) + study.add_trial(past_trial) + study.report_preference(study.trials[:-1], study.trials[-1]) + + trial = study.ask() + trial.suggest_categorical("x", ["A", "B", "C"]) + + +@parametrize_sampler +def test_sample_mixed(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + + for i in range(5): + past_trial = create_trial( + state=TrialState.RUNNING, + params={"x": 1.0, "y": 1, "z": "A"}, + distributions={ + "x": FloatDistribution(0, 10), + "y": IntDistribution(0, 10), + "z": CategoricalDistribution(["A", "B", "C"]), + }, + ) + study.add_trial(past_trial) + study.report_preference(study.trials[:-1], study.trials[-1]) + + trial = study.ask() + trial.suggest_float("x", 0, 10) + trial.suggest_int("y", 0, 10) + trial.suggest_categorical("z", ["A", "B", "C"]) + + +@parametrize_sampler +def test_sample_first_trial(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + trial = study.ask() + trial.suggest_float("x", 0, 10) + + +@parametrize_sampler +def test_sample_dynamic_search_space(sampler_class: Callable[[], BaseSampler]) -> None: + study = create_study(n_generate=4, sampler=sampler_class()) + + for i in range(5): + past_trial = create_trial( + state=TrialState.RUNNING, + params={"x": 1.0}, + distributions={"x": FloatDistribution(0, 10)}, + ) + study.add_trial(past_trial) + study.report_preference(study.trials[:-1], study.trials[-1]) + + trial = study.ask() + trial.suggest_float("x", -100, 100) From 47539c37416bd3eaa45f6b7ba56b80adb0a42790 Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 22 Sep 2023 15:32:44 +0900 Subject: [PATCH 099/104] fix by review --- optuna_dashboard/ts/action.ts | 28 ++++++++++++++----- optuna_dashboard/ts/apiClient.ts | 6 ++-- .../ts/components/PreferenceHistory.tsx | 10 +++---- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index a082d5fd..eb12abb1 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -589,17 +589,21 @@ export const actionCreator = () => { } const updatePreference = ( - study_id: number, + studyId: number, candidates: number[], clicked: number ) => { - reportPreferenceAPI(study_id, candidates, clicked).catch((err) => { - const reason = err.response?.data.reason - enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { - variant: "error", + reportPreferenceAPI(studyId, candidates, clicked) + .then(() => { + updateStudyDetail(studyId) + }) + .catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { + variant: "error", + }) + console.log(err) }) - console.log(err) - }) } const skipPreferentialTrial = (studyId: number, trialId: number) => { @@ -640,6 +644,12 @@ export const actionCreator = () => { newStudy.preference_history = newStudy.preference_history?.map((h) => h.id === historyId ? { ...h, is_removed: true } : h ) + const removed = newStudy.preference_history + ?.filter((h) => h.id === historyId) + .pop()?.preferences + newStudy.preferences = newStudy.preferences?.filter( + (p) => !removed?.some((r) => r[0] === p[0] && r[1] === p[1]) + ) setStudyDetailState(studyId, newStudy) }) .catch((err) => { @@ -658,6 +668,10 @@ export const actionCreator = () => { newStudy.preference_history = newStudy.preference_history?.map((h) => h.id === historyId ? { ...h, is_removed: false } : h ) + const restored = newStudy.preference_history + ?.filter((h) => h.id === historyId) + .pop()?.preferences + newStudy.preferences = newStudy.preferences?.concat(restored ?? []) setStudyDetailState(studyId, newStudy) }) .catch((err) => { diff --git a/optuna_dashboard/ts/apiClient.ts b/optuna_dashboard/ts/apiClient.ts index d6b3c77a..c0065bd3 100644 --- a/optuna_dashboard/ts/apiClient.ts +++ b/optuna_dashboard/ts/apiClient.ts @@ -55,7 +55,7 @@ const convertTrialResponse = (res: TrialResponse): Trial => { } } -interface PreferenceHistoryResponce { +interface PreferenceHistoryResponse { history: { id: string candidates: number[] @@ -68,7 +68,7 @@ interface PreferenceHistoryResponce { } const convertPreferenceHistory = ( - res: PreferenceHistoryResponce + res: PreferenceHistoryResponse ): PreferenceHistory => { return { id: res.history.id, @@ -97,7 +97,7 @@ interface StudyDetailResponse { objective_names?: string[] form_widgets?: FormWidgets preferences?: [number, number][] - preference_history?: PreferenceHistoryResponce[] + preference_history?: PreferenceHistoryResponse[] plotly_graph_objects: PlotlyGraphObject[] feedback_component_type: FeedbackComponentType skipped_trial_numbers?: number[] diff --git a/optuna_dashboard/ts/components/PreferenceHistory.tsx b/optuna_dashboard/ts/components/PreferenceHistory.tsx index a29f84b8..6aa67317 100644 --- a/optuna_dashboard/ts/components/PreferenceHistory.tsx +++ b/optuna_dashboard/ts/components/PreferenceHistory.tsx @@ -163,8 +163,8 @@ const CandidateTrial: FC<{ const ChoiceTrials: FC<{ choice: PreferenceHistory trials: Trial[] - study_id: number -}> = ({ choice, trials, study_id }) => { + studyId: number +}> = ({ choice, trials, studyId }) => { const [isRemoved, setRemoved] = useState(choice.is_removed) const theme = useTheme() const worst_trials = new Set([choice.clicked]) @@ -198,7 +198,7 @@ const ChoiceTrials: FC<{ disabled={!isRemoved} onClick={() => { setRemoved(false) - action.restorePreferentialHistory(study_id, choice.id) + action.restorePreferentialHistory(studyId, choice.id) }} sx={{ margin: `auto ${theme.spacing(2)}`, @@ -211,7 +211,7 @@ const ChoiceTrials: FC<{ disabled={isRemoved} onClick={() => { setRemoved(true) - action.removePreferentialHistory(study_id, choice.id) + action.removePreferentialHistory(studyId, choice.id) }} sx={{ margin: `auto ${theme.spacing(2)}`, @@ -279,7 +279,7 @@ export const PreferenceHistory: FC<{ studyDetail: StudyDetail | null }> = ({ key={choice.id} choice={choice} trials={studyDetail.trials} - study_id={studyDetail.id} + studyId={studyDetail.id} /> ))} From 79161e98cba8ce1d80e01b8fe7a88e8030ed5e49 Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Fri, 22 Sep 2023 15:53:00 +0900 Subject: [PATCH 100/104] Add botorch to optional install libraries --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 9b7ce731..4eff6f1f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,6 +51,7 @@ test = [ optional = [ "streamlit", "boto3", + "botorch", ] From 1c4037ffce229e69fdfec852956ce29f49a33d5e Mon Sep 17 00:00:00 2001 From: Naoto Mizuno Date: Fri, 22 Sep 2023 16:34:20 +0900 Subject: [PATCH 101/104] Skip test for Python 3.7 --- python_tests/preferential/test_samplers.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/python_tests/preferential/test_samplers.py b/python_tests/preferential/test_samplers.py index 00b74ded..2de8864c 100644 --- a/python_tests/preferential/test_samplers.py +++ b/python_tests/preferential/test_samplers.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +import sys import optuna from optuna import create_trial @@ -10,12 +11,26 @@ from optuna.distributions import IntDistribution from optuna.samplers import BaseSampler from optuna.trial import TrialState from optuna_dashboard.preferential import create_study -from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler import pytest +if sys.version_info >= (3, 8): + from optuna_dashboard.preferential.samplers.gp import PreferentialGPSampler +else: + PreferentialGPSampler = None + + parametrize_sampler = pytest.mark.parametrize( - "sampler_class", [optuna.samplers.RandomSampler, PreferentialGPSampler] + "sampler_class", + [ + optuna.samplers.RandomSampler, + pytest.param( + PreferentialGPSampler, + marks=pytest.mark.skipif( + sys.version_info < (3, 8), reason="BoTorch dropped Python3.7 support" + ), + ), + ], ) From 7e06672799ed4d171edb1887b079aa0858264e3c Mon Sep 17 00:00:00 2001 From: moririn2528 Date: Fri, 22 Sep 2023 17:00:26 +0900 Subject: [PATCH 102/104] fit figure size --- optuna_dashboard/ts/components/ArtifactCardMedia.tsx | 3 +++ optuna_dashboard/ts/components/PreferentialGraph.tsx | 10 +++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/optuna_dashboard/ts/components/ArtifactCardMedia.tsx b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx index a994aa41..ebbbd751 100644 --- a/optuna_dashboard/ts/components/ArtifactCardMedia.tsx +++ b/optuna_dashboard/ts/components/ArtifactCardMedia.tsx @@ -34,6 +34,9 @@ export const ArtifactCardMedia: FC<{ height={height} image={urlPath} alt={artifact.filename} + style={{ + objectFit: "contain", + }} /> ) } diff --git a/optuna_dashboard/ts/components/PreferentialGraph.tsx b/optuna_dashboard/ts/components/PreferentialGraph.tsx index 4f2c1059..ae5981b3 100644 --- a/optuna_dashboard/ts/components/PreferentialGraph.tsx +++ b/optuna_dashboard/ts/components/PreferentialGraph.tsx @@ -91,7 +91,15 @@ const GraphNode: FC> = ({ data, isConnectable }) => { style={{ background: "#555" }} isConnectable={isConnectable} /> - + Date: Fri, 22 Sep 2023 17:16:02 +0900 Subject: [PATCH 103/104] fix by review --- optuna_dashboard/ts/action.ts | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/optuna_dashboard/ts/action.ts b/optuna_dashboard/ts/action.ts index eb12abb1..41afdadb 100644 --- a/optuna_dashboard/ts/action.ts +++ b/optuna_dashboard/ts/action.ts @@ -593,17 +593,13 @@ export const actionCreator = () => { candidates: number[], clicked: number ) => { - reportPreferenceAPI(studyId, candidates, clicked) - .then(() => { - updateStudyDetail(studyId) - }) - .catch((err) => { - const reason = err.response?.data.reason - enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { - variant: "error", - }) - console.log(err) + reportPreferenceAPI(studyId, candidates, clicked).catch((err) => { + const reason = err.response?.data.reason + enqueueSnackbar(`Failed to report preference. Reason: ${reason}`, { + variant: "error", }) + console.log(err) + }) } const skipPreferentialTrial = (studyId: number, trialId: number) => { From 1ff2946d7427a7a55a442403afa8b82b2588cd5e Mon Sep 17 00:00:00 2001 From: hrntsm Date: Tue, 26 Sep 2023 20:58:59 +0900 Subject: [PATCH 104/104] Fix 3dm axis handling --- optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx index a88b9d2c..3c4e2e01 100644 --- a/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx +++ b/optuna_dashboard/ts/components/ThreejsArtifactViewer.tsx @@ -75,10 +75,8 @@ export const ThreejsArtifactViewer: React.FC = ( loader.load(props.src, (object: THREE.Object3D) => { const meshes = object.children as THREE.Mesh[] const rhinoGeometries = meshes.map((mesh) => mesh.geometry) + THREE.Object3D.DEFAULT_UP.set(0, 0, 1) if (rhinoGeometries.length > 0) { - rhinoGeometries.forEach((rhinoGeometry) => { - rhinoGeometry.rotateX(-Math.PI / 4) - }) handleLoadedGeometries(rhinoGeometries) } })