2018-11-17 18:03:33 -08:00
|
|
|
# Copyright 2018 Google LLC
|
|
|
|
#
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
#
|
|
|
|
# https://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
#
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
# limitations under the License.
|
|
|
|
|
2020-06-06 10:51:34 -07:00
|
|
|
# flake8: noqa: F401
|
2020-05-07 17:24:19 -04:00
|
|
|
from .config import config
|
|
|
|
from .api import (
|
|
|
|
ad, # TODO(phawkins): update users to avoid this.
|
|
|
|
argnums_partial, # TODO(phawkins): update Haiku to not use this.
|
|
|
|
checkpoint,
|
|
|
|
curry, # TODO(phawkins): update users to avoid this.
|
Initial version of invertible AD implementation (#3232)
This is a prototype implementation of the memory-efficient VJP method
for invertible function. The general idea is that thanks to
invertibility, we don't have to memoize any intermediate primal values,
but can simply reconstruct them in lock-step with gradient computation.
The API is such that the only thing a user has to do, is decorate a
function with `@invertible`, which will make AD apply the more efficient
transpose than usual.
The current version is expressive enough to support e.g. the Reversible
ResNet, but there are still some caveats:
- The definition of "invertible" function is a one that produces a jaxpr
that can be inverted correctly if only we iterate over its equations
in reverse. This is a bit strict, because users generally don't have
too much control over that, and there are functions that produce
jaxprs which will be treated as invertible when one topological
ordering of equations is used, while they will be considered
non-invertible for other valid orderings.
- It doesn't follow the usual jvp + transpose path, and it turns out
that zero argument pruning in JVPTrace makes it pretty much impossible
to implement correctly.
- `custom_ivjp` is an initial-style primitive.
- Invertible reverse-mode implementation (`rev_backward_pass`) assumes
that all the VJPs of primal primitives are jittable (not sure if
that's a problem, but worth pointing out).
- Not having a dedicated linearization pass makes the JVP of
`custom_ivjp` inefficient if it is being staged out.
2020-06-15 12:35:06 +02:00
|
|
|
custom_ivjp,
|
2020-05-07 17:24:19 -04:00
|
|
|
custom_gradient,
|
|
|
|
custom_jvp,
|
|
|
|
custom_vjp,
|
|
|
|
custom_transforms,
|
|
|
|
defjvp,
|
|
|
|
defjvp_all,
|
|
|
|
defvjp,
|
|
|
|
defvjp_all,
|
|
|
|
device_count,
|
|
|
|
device_get,
|
|
|
|
device_put,
|
|
|
|
devices,
|
|
|
|
disable_jit,
|
|
|
|
eval_shape,
|
|
|
|
flatten_fun_nokwargs, # TODO(phawkins): update users to avoid this.
|
|
|
|
grad,
|
|
|
|
hessian,
|
|
|
|
host_count,
|
|
|
|
host_id,
|
|
|
|
host_ids,
|
Initial version of invertible AD implementation (#3232)
This is a prototype implementation of the memory-efficient VJP method
for invertible function. The general idea is that thanks to
invertibility, we don't have to memoize any intermediate primal values,
but can simply reconstruct them in lock-step with gradient computation.
The API is such that the only thing a user has to do, is decorate a
function with `@invertible`, which will make AD apply the more efficient
transpose than usual.
The current version is expressive enough to support e.g. the Reversible
ResNet, but there are still some caveats:
- The definition of "invertible" function is a one that produces a jaxpr
that can be inverted correctly if only we iterate over its equations
in reverse. This is a bit strict, because users generally don't have
too much control over that, and there are functions that produce
jaxprs which will be treated as invertible when one topological
ordering of equations is used, while they will be considered
non-invertible for other valid orderings.
- It doesn't follow the usual jvp + transpose path, and it turns out
that zero argument pruning in JVPTrace makes it pretty much impossible
to implement correctly.
- `custom_ivjp` is an initial-style primitive.
- Invertible reverse-mode implementation (`rev_backward_pass`) assumes
that all the VJPs of primal primitives are jittable (not sure if
that's a problem, but worth pointing out).
- Not having a dedicated linearization pass makes the JVP of
`custom_ivjp` inefficient if it is being staged out.
2020-06-15 12:35:06 +02:00
|
|
|
invertible,
|
2020-05-07 17:24:19 -04:00
|
|
|
jacobian,
|
|
|
|
jacfwd,
|
|
|
|
jacrev,
|
|
|
|
jit,
|
|
|
|
jvp,
|
|
|
|
local_device_count,
|
|
|
|
local_devices,
|
|
|
|
linearize,
|
|
|
|
make_jaxpr,
|
|
|
|
mask,
|
|
|
|
partial, # TODO(phawkins): update callers to use functools.partial.
|
|
|
|
pmap,
|
|
|
|
pxla, # TODO(phawkins): update users to avoid this.
|
|
|
|
remat,
|
|
|
|
shapecheck,
|
|
|
|
ShapedArray,
|
|
|
|
ShapeDtypeStruct,
|
|
|
|
soft_pmap,
|
|
|
|
# TODO(phawkins): hide tree* functions from jax, update callers to use
|
|
|
|
# jax.tree_util.
|
|
|
|
treedef_is_leaf,
|
|
|
|
tree_flatten,
|
|
|
|
tree_leaves,
|
|
|
|
tree_map,
|
|
|
|
tree_multimap,
|
|
|
|
tree_structure,
|
|
|
|
tree_transpose,
|
|
|
|
tree_unflatten,
|
|
|
|
value_and_grad,
|
|
|
|
vjp,
|
|
|
|
vmap,
|
|
|
|
xla, # TODO(phawkins): update users to avoid this.
|
|
|
|
xla_computation,
|
|
|
|
)
|
2020-05-19 20:40:03 +01:00
|
|
|
from .version import __version__
|
2020-05-07 17:24:19 -04:00
|
|
|
|
2020-05-19 20:40:03 +01:00
|
|
|
# These submodules are separate because they are in an import cycle with
|
|
|
|
# jax and rely on the names imported above.
|
|
|
|
from . import lax
|
|
|
|
from . import nn
|
|
|
|
from . import random
|
2020-05-07 17:24:19 -04:00
|
|
|
|
|
|
|
def _init():
|
|
|
|
import os
|
|
|
|
os.environ.setdefault('TF_CPP_MIN_LOG_LEVEL', '1')
|
|
|
|
|
2020-05-19 20:40:03 +01:00
|
|
|
from . import numpy # side-effecting import sets up operator overloads
|
2020-05-08 10:04:19 -04:00
|
|
|
|
2020-05-07 17:24:19 -04:00
|
|
|
_init()
|
|
|
|
del _init
|