This commit is contained in:
2024-11-29 18:15:30 +00:00
parent 40aade2d8e
commit bc9415586e
5298 changed files with 1938676 additions and 80 deletions

View File

@ -0,0 +1,20 @@
from .main import minimize
from .utils import show_versions
# PEP0440 compatible formatted version, see:
# https://www.python.org/dev/peps/pep-0440/
#
# Final release markers:
# X.Y.0 # For first release after an increment in Y
# X.Y.Z # For bugfix releases
#
# Admissible pre-release markers:
# X.YaN # Alpha release
# X.YbN # Beta release
# X.YrcN # Release Candidate
#
# Dev branch marker is: 'X.Y.dev' or 'X.Y.devN' where N is an integer.
# 'X.Y.dev0' is the canonical version of 'X.Y.dev'.
__version__ = "1.1.1"
__all__ = ["minimize", "show_versions"]

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,132 @@
import sys
from enum import Enum
import numpy as np
# Exit status.
class ExitStatus(Enum):
"""
Exit statuses.
"""
RADIUS_SUCCESS = 0
TARGET_SUCCESS = 1
FIXED_SUCCESS = 2
CALLBACK_SUCCESS = 3
FEASIBLE_SUCCESS = 4
MAX_EVAL_WARNING = 5
MAX_ITER_WARNING = 6
INFEASIBLE_ERROR = -1
LINALG_ERROR = -2
class Options(str, Enum):
"""
Options.
"""
DEBUG = "debug"
FEASIBILITY_TOL = "feasibility_tol"
FILTER_SIZE = "filter_size"
HISTORY_SIZE = "history_size"
MAX_EVAL = "maxfev"
MAX_ITER = "maxiter"
NPT = "nb_points"
RHOBEG = "radius_init"
RHOEND = "radius_final"
SCALE = "scale"
STORE_HISTORY = "store_history"
TARGET = "target"
VERBOSE = "disp"
class Constants(str, Enum):
"""
Constants.
"""
DECREASE_RADIUS_FACTOR = "decrease_radius_factor"
INCREASE_RADIUS_FACTOR = "increase_radius_factor"
INCREASE_RADIUS_THRESHOLD = "increase_radius_threshold"
DECREASE_RADIUS_THRESHOLD = "decrease_radius_threshold"
DECREASE_RESOLUTION_FACTOR = "decrease_resolution_factor"
LARGE_RESOLUTION_THRESHOLD = "large_resolution_threshold"
MODERATE_RESOLUTION_THRESHOLD = "moderate_resolution_threshold"
LOW_RATIO = "low_ratio"
HIGH_RATIO = "high_ratio"
VERY_LOW_RATIO = "very_low_ratio"
PENALTY_INCREASE_THRESHOLD = "penalty_increase_threshold"
PENALTY_INCREASE_FACTOR = "penalty_increase_factor"
SHORT_STEP_THRESHOLD = "short_step_threshold"
LOW_RADIUS_FACTOR = "low_radius_factor"
BYRD_OMOJOKUN_FACTOR = "byrd_omojokun_factor"
THRESHOLD_RATIO_CONSTRAINTS = "threshold_ratio_constraints"
LARGE_SHIFT_FACTOR = "large_shift_factor"
LARGE_GRADIENT_FACTOR = "large_gradient_factor"
RESOLUTION_FACTOR = "resolution_factor"
IMPROVE_TCG = "improve_tcg"
# Default options.
DEFAULT_OPTIONS = {
Options.DEBUG.value: False,
Options.FEASIBILITY_TOL.value: np.sqrt(np.finfo(float).eps),
Options.FILTER_SIZE.value: sys.maxsize,
Options.HISTORY_SIZE.value: sys.maxsize,
Options.MAX_EVAL.value: lambda n: 500 * n,
Options.MAX_ITER.value: lambda n: 1000 * n,
Options.NPT.value: lambda n: 2 * n + 1,
Options.RHOBEG.value: 1.0,
Options.RHOEND.value: 1e-6,
Options.SCALE.value: False,
Options.STORE_HISTORY.value: False,
Options.TARGET.value: -np.inf,
Options.VERBOSE.value: False,
}
# Default constants.
DEFAULT_CONSTANTS = {
Constants.DECREASE_RADIUS_FACTOR.value: 0.5,
Constants.INCREASE_RADIUS_FACTOR.value: np.sqrt(2.0),
Constants.INCREASE_RADIUS_THRESHOLD.value: 2.0,
Constants.DECREASE_RADIUS_THRESHOLD.value: 1.4,
Constants.DECREASE_RESOLUTION_FACTOR.value: 0.1,
Constants.LARGE_RESOLUTION_THRESHOLD.value: 250.0,
Constants.MODERATE_RESOLUTION_THRESHOLD.value: 16.0,
Constants.LOW_RATIO.value: 0.1,
Constants.HIGH_RATIO.value: 0.7,
Constants.VERY_LOW_RATIO.value: 0.01,
Constants.PENALTY_INCREASE_THRESHOLD.value: 1.5,
Constants.PENALTY_INCREASE_FACTOR.value: 2.0,
Constants.SHORT_STEP_THRESHOLD.value: 0.5,
Constants.LOW_RADIUS_FACTOR.value: 0.1,
Constants.BYRD_OMOJOKUN_FACTOR.value: 0.8,
Constants.THRESHOLD_RATIO_CONSTRAINTS.value: 2.0,
Constants.LARGE_SHIFT_FACTOR.value: 10.0,
Constants.LARGE_GRADIENT_FACTOR.value: 10.0,
Constants.RESOLUTION_FACTOR.value: 2.0,
Constants.IMPROVE_TCG.value: True,
}
# Printing options.
PRINT_OPTIONS = {
"threshold": 6,
"edgeitems": 2,
"linewidth": sys.maxsize,
"formatter": {
"float_kind": lambda x: np.format_float_scientific(
x,
precision=3,
unique=False,
pad_left=2,
)
},
}
# Constants.
BARRIER = 2.0 ** min(
100,
np.finfo(float).maxexp // 2,
-np.finfo(float).minexp // 2,
)

View File

@ -0,0 +1,14 @@
from .geometry import cauchy_geometry, spider_geometry
from .optim import (
tangential_byrd_omojokun,
constrained_tangential_byrd_omojokun,
normal_byrd_omojokun,
)
__all__ = [
"cauchy_geometry",
"spider_geometry",
"tangential_byrd_omojokun",
"constrained_tangential_byrd_omojokun",
"normal_byrd_omojokun",
]

View File

@ -0,0 +1,387 @@
import inspect
import numpy as np
from ..utils import get_arrays_tol
TINY = np.finfo(float).tiny
def cauchy_geometry(const, grad, curv, xl, xu, delta, debug):
r"""
Maximize approximately the absolute value of a quadratic function subject
to bound constraints in a trust region.
This function solves approximately
.. math::
\max_{s \in \mathbb{R}^n} \quad \bigg\lvert c + g^{\mathsf{T}} s +
\frac{1}{2} s^{\mathsf{T}} H s \bigg\rvert \quad \text{s.t.} \quad
\left\{ \begin{array}{l}
l \le s \le u,\\
\lVert s \rVert \le \Delta,
\end{array} \right.
by maximizing the objective function along the constrained Cauchy
direction.
Parameters
----------
const : float
Constant :math:`c` as shown above.
grad : `numpy.ndarray`, shape (n,)
Gradient :math:`g` as shown above.
curv : callable
Curvature of :math:`H` along any vector.
``curv(s) -> float``
returns :math:`s^{\mathsf{T}} H s`.
xl : `numpy.ndarray`, shape (n,)
Lower bounds :math:`l` as shown above.
xu : `numpy.ndarray`, shape (n,)
Upper bounds :math:`u` as shown above.
delta : float
Trust-region radius :math:`\Delta` as shown above.
debug : bool
Whether to make debugging tests during the execution.
Returns
-------
`numpy.ndarray`, shape (n,)
Approximate solution :math:`s`.
Notes
-----
This function is described as the first alternative in Section 6.5 of [1]_.
It is assumed that the origin is feasible with respect to the bound
constraints and that `delta` is finite and positive.
References
----------
.. [1] T. M. Ragonneau. *Model-Based Derivative-Free Optimization Methods
and Software*. PhD thesis, Department of Applied Mathematics, The Hong
Kong Polytechnic University, Hong Kong, China, 2022. URL:
https://theses.lib.polyu.edu.hk/handle/200/12294.
"""
if debug:
assert isinstance(const, float)
assert isinstance(grad, np.ndarray) and grad.ndim == 1
assert inspect.signature(curv).bind(grad)
assert isinstance(xl, np.ndarray) and xl.shape == grad.shape
assert isinstance(xu, np.ndarray) and xu.shape == grad.shape
assert isinstance(delta, float)
assert isinstance(debug, bool)
tol = get_arrays_tol(xl, xu)
assert np.all(xl <= tol)
assert np.all(xu >= -tol)
assert np.isfinite(delta) and delta > 0.0
xl = np.minimum(xl, 0.0)
xu = np.maximum(xu, 0.0)
# To maximize the absolute value of a quadratic function, we maximize the
# function itself or its negative, and we choose the solution that provides
# the largest function value.
step1, q_val1 = _cauchy_geom(const, grad, curv, xl, xu, delta, debug)
step2, q_val2 = _cauchy_geom(
-const,
-grad,
lambda x: -curv(x),
xl,
xu,
delta,
debug,
)
step = step1 if abs(q_val1) >= abs(q_val2) else step2
if debug:
assert np.all(xl <= step)
assert np.all(step <= xu)
assert np.linalg.norm(step) < 1.1 * delta
return step
def spider_geometry(const, grad, curv, xpt, xl, xu, delta, debug):
r"""
Maximize approximately the absolute value of a quadratic function subject
to bound constraints in a trust region.
This function solves approximately
.. math::
\max_{s \in \mathbb{R}^n} \quad \bigg\lvert c + g^{\mathsf{T}} s +
\frac{1}{2} s^{\mathsf{T}} H s \bigg\rvert \quad \text{s.t.} \quad
\left\{ \begin{array}{l}
l \le s \le u,\\
\lVert s \rVert \le \Delta,
\end{array} \right.
by maximizing the objective function along given straight lines.
Parameters
----------
const : float
Constant :math:`c` as shown above.
grad : `numpy.ndarray`, shape (n,)
Gradient :math:`g` as shown above.
curv : callable
Curvature of :math:`H` along any vector.
``curv(s) -> float``
returns :math:`s^{\mathsf{T}} H s`.
xpt : `numpy.ndarray`, shape (n, npt)
Points defining the straight lines. The straight lines considered are
the ones passing through the origin and the points in `xpt`.
xl : `numpy.ndarray`, shape (n,)
Lower bounds :math:`l` as shown above.
xu : `numpy.ndarray`, shape (n,)
Upper bounds :math:`u` as shown above.
delta : float
Trust-region radius :math:`\Delta` as shown above.
debug : bool
Whether to make debugging tests during the execution.
Returns
-------
`numpy.ndarray`, shape (n,)
Approximate solution :math:`s`.
Notes
-----
This function is described as the second alternative in Section 6.5 of
[1]_. It is assumed that the origin is feasible with respect to the bound
constraints and that `delta` is finite and positive.
References
----------
.. [1] T. M. Ragonneau. *Model-Based Derivative-Free Optimization Methods
and Software*. PhD thesis, Department of Applied Mathematics, The Hong
Kong Polytechnic University, Hong Kong, China, 2022. URL:
https://theses.lib.polyu.edu.hk/handle/200/12294.
"""
if debug:
assert isinstance(const, float)
assert isinstance(grad, np.ndarray) and grad.ndim == 1
assert inspect.signature(curv).bind(grad)
assert (
isinstance(xpt, np.ndarray)
and xpt.ndim == 2
and xpt.shape[0] == grad.size
)
assert isinstance(xl, np.ndarray) and xl.shape == grad.shape
assert isinstance(xu, np.ndarray) and xu.shape == grad.shape
assert isinstance(delta, float)
assert isinstance(debug, bool)
tol = get_arrays_tol(xl, xu)
assert np.all(xl <= tol)
assert np.all(xu >= -tol)
assert np.isfinite(delta) and delta > 0.0
xl = np.minimum(xl, 0.0)
xu = np.maximum(xu, 0.0)
# Iterate through the straight lines.
step = np.zeros_like(grad)
q_val = const
s_norm = np.linalg.norm(xpt, axis=0)
# Set alpha_xl to the step size for the lower-bound constraint and
# alpha_xu to the step size for the upper-bound constraint.
# xl.shape = (N,)
# xpt.shape = (N, M)
# i_xl_pos.shape = (M, N)
i_xl_pos = (xl > -np.inf) & (xpt.T > -TINY * xl)
i_xl_neg = (xl > -np.inf) & (xpt.T < TINY * xl)
i_xu_pos = (xu < np.inf) & (xpt.T > TINY * xu)
i_xu_neg = (xu < np.inf) & (xpt.T < -TINY * xu)
# (M, N)
alpha_xl_pos = np.atleast_2d(
np.broadcast_to(xl, i_xl_pos.shape)[i_xl_pos] / xpt.T[i_xl_pos]
)
# (M,)
alpha_xl_pos = np.max(alpha_xl_pos, axis=1, initial=-np.inf)
# make sure it's (M,)
alpha_xl_pos = np.broadcast_to(np.atleast_1d(alpha_xl_pos), xpt.shape[1])
alpha_xl_neg = np.atleast_2d(
np.broadcast_to(xl, i_xl_neg.shape)[i_xl_neg] / xpt.T[i_xl_neg]
)
alpha_xl_neg = np.max(alpha_xl_neg, axis=1, initial=np.inf)
alpha_xl_neg = np.broadcast_to(np.atleast_1d(alpha_xl_neg), xpt.shape[1])
alpha_xu_neg = np.atleast_2d(
np.broadcast_to(xu, i_xu_neg.shape)[i_xu_neg] / xpt.T[i_xu_neg]
)
alpha_xu_neg = np.max(alpha_xu_neg, axis=1, initial=-np.inf)
alpha_xu_neg = np.broadcast_to(np.atleast_1d(alpha_xu_neg), xpt.shape[1])
alpha_xu_pos = np.atleast_2d(
np.broadcast_to(xu, i_xu_pos.shape)[i_xu_pos] / xpt.T[i_xu_pos]
)
alpha_xu_pos = np.max(alpha_xu_pos, axis=1, initial=np.inf)
alpha_xu_pos = np.broadcast_to(np.atleast_1d(alpha_xu_pos), xpt.shape[1])
for k in range(xpt.shape[1]):
# Set alpha_tr to the step size for the trust-region constraint.
if s_norm[k] > TINY * delta:
alpha_tr = max(delta / s_norm[k], 0.0)
else:
# The current straight line is basically zero.
continue
alpha_bd_pos = max(min(alpha_xu_pos[k], alpha_xl_neg[k]), 0.0)
alpha_bd_neg = min(max(alpha_xl_pos[k], alpha_xu_neg[k]), 0.0)
# Set alpha_quad_pos and alpha_quad_neg to the step size to the extrema
# of the quadratic function along the positive and negative directions.
grad_step = grad @ xpt[:, k]
curv_step = curv(xpt[:, k])
if (
grad_step >= 0.0
and curv_step < -TINY * grad_step
or grad_step <= 0.0
and curv_step > -TINY * grad_step
):
alpha_quad_pos = max(-grad_step / curv_step, 0.0)
else:
alpha_quad_pos = np.inf
if (
grad_step >= 0.0
and curv_step > TINY * grad_step
or grad_step <= 0.0
and curv_step < TINY * grad_step
):
alpha_quad_neg = min(-grad_step / curv_step, 0.0)
else:
alpha_quad_neg = -np.inf
# Select the step that provides the largest value of the objective
# function if it improves the current best. The best positive step is
# either the one that reaches the constraints or the one that reaches
# the extremum of the objective function along the current direction
# (only possible if the resulting step is feasible). We test both, and
# we perform similar calculations along the negative step.
# N.B.: we select the largest possible step among all the ones that
# maximize the objective function. This is to avoid returning the zero
# step in some extreme cases.
alpha_pos = min(alpha_tr, alpha_bd_pos)
alpha_neg = max(-alpha_tr, alpha_bd_neg)
q_val_pos = (
const + alpha_pos * grad_step + 0.5 * alpha_pos**2.0 * curv_step
)
q_val_neg = (
const + alpha_neg * grad_step + 0.5 * alpha_neg**2.0 * curv_step
)
if alpha_quad_pos < alpha_pos:
q_val_quad_pos = (
const
+ alpha_quad_pos * grad_step
+ 0.5 * alpha_quad_pos**2.0 * curv_step
)
if abs(q_val_quad_pos) > abs(q_val_pos):
alpha_pos = alpha_quad_pos
q_val_pos = q_val_quad_pos
if alpha_quad_neg > alpha_neg:
q_val_quad_neg = (
const
+ alpha_quad_neg * grad_step
+ 0.5 * alpha_quad_neg**2.0 * curv_step
)
if abs(q_val_quad_neg) > abs(q_val_neg):
alpha_neg = alpha_quad_neg
q_val_neg = q_val_quad_neg
if abs(q_val_pos) >= abs(q_val_neg) and abs(q_val_pos) > abs(q_val):
step = np.clip(alpha_pos * xpt[:, k], xl, xu)
q_val = q_val_pos
elif abs(q_val_neg) > abs(q_val_pos) and abs(q_val_neg) > abs(q_val):
step = np.clip(alpha_neg * xpt[:, k], xl, xu)
q_val = q_val_neg
if debug:
assert np.all(xl <= step)
assert np.all(step <= xu)
assert np.linalg.norm(step) < 1.1 * delta
return step
def _cauchy_geom(const, grad, curv, xl, xu, delta, debug):
"""
Same as `bound_constrained_cauchy_step` without the absolute value.
"""
# Calculate the initial active set.
fixed_xl = (xl < 0.0) & (grad > 0.0)
fixed_xu = (xu > 0.0) & (grad < 0.0)
# Calculate the Cauchy step.
cauchy_step = np.zeros_like(grad)
cauchy_step[fixed_xl] = xl[fixed_xl]
cauchy_step[fixed_xu] = xu[fixed_xu]
if np.linalg.norm(cauchy_step) > delta:
working = fixed_xl | fixed_xu
while True:
# Calculate the Cauchy step for the directions in the working set.
g_norm = np.linalg.norm(grad[working])
delta_reduced = np.sqrt(
delta**2.0 - cauchy_step[~working] @ cauchy_step[~working]
)
if g_norm > TINY * abs(delta_reduced):
mu = max(delta_reduced / g_norm, 0.0)
else:
break
cauchy_step[working] = mu * grad[working]
# Update the working set.
fixed_xl = working & (cauchy_step < xl)
fixed_xu = working & (cauchy_step > xu)
if not np.any(fixed_xl) and not np.any(fixed_xu):
# Stop the calculations as the Cauchy step is now feasible.
break
cauchy_step[fixed_xl] = xl[fixed_xl]
cauchy_step[fixed_xu] = xu[fixed_xu]
working = working & ~(fixed_xl | fixed_xu)
# Calculate the step that maximizes the quadratic along the Cauchy step.
grad_step = grad @ cauchy_step
if grad_step >= 0.0:
# Set alpha_tr to the step size for the trust-region constraint.
s_norm = np.linalg.norm(cauchy_step)
if s_norm > TINY * delta:
alpha_tr = max(delta / s_norm, 0.0)
else:
# The Cauchy step is basically zero.
alpha_tr = 0.0
# Set alpha_quad to the step size for the maximization problem.
curv_step = curv(cauchy_step)
if curv_step < -TINY * grad_step:
alpha_quad = max(-grad_step / curv_step, 0.0)
else:
alpha_quad = np.inf
# Set alpha_bd to the step size for the bound constraints.
i_xl = (xl > -np.inf) & (cauchy_step < TINY * xl)
i_xu = (xu < np.inf) & (cauchy_step > TINY * xu)
alpha_xl = np.min(xl[i_xl] / cauchy_step[i_xl], initial=np.inf)
alpha_xu = np.min(xu[i_xu] / cauchy_step[i_xu], initial=np.inf)
alpha_bd = min(alpha_xl, alpha_xu)
# Calculate the solution and the corresponding function value.
alpha = min(alpha_tr, alpha_quad, alpha_bd)
step = np.clip(alpha * cauchy_step, xl, xu)
q_val = const + alpha * grad_step + 0.5 * alpha**2.0 * curv_step
else:
# This case is never reached in exact arithmetic. It prevents this
# function to return a step that decreases the objective function.
step = np.zeros_like(grad)
q_val = const
if debug:
assert np.all(xl <= step)
assert np.all(step <= xu)
assert np.linalg.norm(step) < 1.1 * delta
return step, q_val

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,18 @@
from .exceptions import (
MaxEvalError,
TargetSuccess,
CallbackSuccess,
FeasibleSuccess,
)
from .math import get_arrays_tol, exact_1d_array
from .versions import show_versions
__all__ = [
"MaxEvalError",
"TargetSuccess",
"CallbackSuccess",
"FeasibleSuccess",
"get_arrays_tol",
"exact_1d_array",
"show_versions",
]

View File

@ -0,0 +1,22 @@
class MaxEvalError(Exception):
"""
Exception raised when the maximum number of evaluations is reached.
"""
class TargetSuccess(Exception):
"""
Exception raised when the target value is reached.
"""
class CallbackSuccess(StopIteration):
"""
Exception raised when the callback function raises a ``StopIteration``.
"""
class FeasibleSuccess(Exception):
"""
Exception raised when a feasible point of a feasible problem is found.
"""

View File

@ -0,0 +1,77 @@
import numpy as np
EPS = np.finfo(float).eps
def get_arrays_tol(*arrays):
"""
Get a relative tolerance for a set of arrays.
Parameters
----------
*arrays: tuple
Set of `numpy.ndarray` to get the tolerance for.
Returns
-------
float
Relative tolerance for the set of arrays.
Raises
------
ValueError
If no array is provided.
"""
if len(arrays) == 0:
raise ValueError("At least one array must be provided.")
size = max(array.size for array in arrays)
weight = max(
np.max(np.abs(array[np.isfinite(array)]), initial=1.0)
for array in arrays
)
return 10.0 * EPS * max(size, 1.0) * weight
def exact_1d_array(x, message):
"""
Preprocess a 1-dimensional array.
Parameters
----------
x : array_like
Array to be preprocessed.
message : str
Error message if `x` cannot be interpreter as a 1-dimensional array.
Returns
-------
`numpy.ndarray`
Preprocessed array.
"""
x = np.atleast_1d(np.squeeze(x)).astype(float)
if x.ndim != 1:
raise ValueError(message)
return x
def exact_2d_array(x, message):
"""
Preprocess a 2-dimensional array.
Parameters
----------
x : array_like
Array to be preprocessed.
message : str
Error message if `x` cannot be interpreter as a 2-dimensional array.
Returns
-------
`numpy.ndarray`
Preprocessed array.
"""
x = np.atleast_2d(x).astype(float)
if x.ndim != 2:
raise ValueError(message)
return x

View File

@ -0,0 +1,67 @@
import os
import platform
import sys
from importlib.metadata import PackageNotFoundError, version
def _get_sys_info():
"""
Get useful system information.
Returns
-------
dict
Useful system information.
"""
return {
"python": sys.version.replace(os.linesep, " "),
"executable": sys.executable,
"machine": platform.platform(),
}
def _get_deps_info():
"""
Get the versions of the dependencies.
Returns
-------
dict
Versions of the dependencies.
"""
deps = ["cobyqa", "numpy", "scipy", "setuptools", "pip"]
deps_info = {}
for module in deps:
try:
deps_info[module] = version(module)
except PackageNotFoundError:
deps_info[module] = None
return deps_info
def show_versions():
"""
Display useful system and dependencies information.
When reporting issues, please include this information.
"""
print("System settings")
print("---------------")
sys_info = _get_sys_info()
print(
"\n".join(
f"{k:>{max(map(len, sys_info.keys())) + 1}}: {v}"
for k, v in sys_info.items()
)
)
print()
print("Python dependencies")
print("-------------------")
deps_info = _get_deps_info()
print(
"\n".join(
f"{k:>{max(map(len, deps_info.keys())) + 1}}: {v}"
for k, v in deps_info.items()
)
)