Index

A

add_meta_graph_and_variables() (SavedModelBuilder method) add_sync_op() (PSSynchronizer method) all_update_ops() (GraphItem property) AllReduce (class in autodist.strategy.all_reduce_strategy) AllReduceSynchronizer (class in autodist.kernel.synchronization.all_reduce_synchronizer) (class in autodist.proto.synchronizers_pb2) (Strategy.Node attribute) apply() (Kernel class method) (Replicator class method) (VariablePartitioner class method) as_default() (GraphItem method) (Remapper method) assign() (PSGradientTaskAssigner method) assign_cluster_information() (AllReduceSynchronizer method) (PSSynchronizer method) (Synchronizer method) AUTO (AllReduceSynchronizer attribute) autodist module AutoDist (class in autodist.autodist) autodist.autodist module autodist.checkpoint module autodist.checkpoint.saved_model_builder module autodist.checkpoint.saver module autodist.cluster module autodist.const module autodist.coordinator module autodist.graph_item module autodist.kernel module autodist.kernel.common module autodist.kernel.common.op_info module autodist.kernel.common.proxy_variable module autodist.kernel.common.utils module autodist.kernel.common.variable_utils module autodist.kernel.device module autodist.kernel.device.resolver module autodist.kernel.graph_transformer module autodist.kernel.kernel module autodist.kernel.partitioner module autodist.kernel.replicator module autodist.kernel.synchronization module autodist.kernel.synchronization.all_reduce_synchronizer module

autodist.kernel.synchronization.collective_key module autodist.kernel.synchronization.compressor module autodist.kernel.synchronization.ps_synchronizer module autodist.kernel.synchronization.synchronizer module autodist.patch module autodist.proto module autodist.proto.graphitem_pb2 module autodist.proto.strategy_pb2 module autodist.proto.synchronizers_pb2 module autodist.remapper module autodist.resource_spec module autodist.runner module autodist.simulator module autodist.strategy module autodist.strategy.all_reduce_strategy module autodist.strategy.base module autodist.strategy.parallax_strategy module autodist.strategy.partitioned_all_reduce_strategy module autodist.strategy.partitioned_ps_strategy module autodist.strategy.ps_lb_strategy module autodist.strategy.ps_strategy module autodist.strategy.random_axis_partition_all_reduce_strategy module autodist.strategy.uneven_partition_ps_strategy module autodist.utils module autodist.utils.logging module autodist.utils.network module autodist.utils.server_starter module autodist.utils.visualization_util module AUTODIST_DEBUG_REMOTE (ENV attribute) AUTODIST_INTERNAL_TF (ENV attribute) AUTODIST_IS_TESTING (ENV attribute) AUTODIST_MIN_LOG_LEVEL (ENV attribute) AUTODIST_PATCH_TF (ENV attribute) AUTODIST_STRATEGY_ID (ENV attribute) AUTODIST_WORKER (ENV attribute) axis() (PartitionerConfig property)

B

between_graph_apply() (AllReduceSynchronizer method) (PSSynchronizer method) (Synchronizer method) build() (AllReduce method) (Parallax method) (PartitionedAR method) (PartitionedPS method) (PS method) (PSLoadBalancing method) (RandomAxisPartitionAR method) (StrategyBuilder method) (UnevenPartitionedPS method)

build_strategy() (AutoDist method) byte_size_load_fn() (in module autodist.strategy.partitioned_ps_strategy) (in module autodist.strategy.ps_lb_strategy) (in module autodist.strategy.uneven_partition_ps_strategy) ByteSize() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method)

C

cached_property() (in module autodist.graph_item) chief() (ResourceSpec property) Clear() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) clear() (SSHConfigMap method) Clear() (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) ClearExtension() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) ClearField() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) Cluster (class in autodist.cluster) CollectiveKey (class in autodist.kernel.synchronization.collective_key) CollectiveOpsConfig (class in autodist.kernel.synchronization.compressor) compile() (StrategyCompiler method)

Compressor (AllReduceSynchronizer attribute) compressor (AllReduceSynchronizer attribute) Compressor (class in autodist.kernel.synchronization.compressor) CompressorEF (class in autodist.kernel.synchronization.compressor) Connectivity (class in autodist.resource_spec) Coordinator (class in autodist.coordinator) copy() (Info method) (SSHConfigMap method) (Strategy method) copy_gradient_info_from() (GraphItem method) CopyFrom() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) count() (SSHConfig method) CPU (DeviceType attribute) cpu_devices() (ResourceSpec property) CPU_TO_GPU (Connectivity attribute) create() (AllReduceSynchronizer class method) (Compressor class method) (CompressorEF class method) (HorovodCompressor class method) (HorovodCompressorEF class method) (NoneCompressor class method) (PSSynchronizer class method) (Synchronizer class method) create_distributed_session() (AutoDist method) critical() (in module autodist.utils.logging)

D

debug() (in module autodist.utils.logging) DESCRIPTOR (AllReduceSynchronizer attribute) (GraphItem attribute) (GraphItem.GradTargetPairsEntry attribute) (GraphItem.Info attribute) (PSSynchronizer attribute) (Strategy attribute) (Strategy.GraphConfig attribute) (Strategy.Node attribute) deserialize() (GraphItem class method) (PartitionerConfig method) (Strategy class method)

DeviceResolver (class in autodist.kernel.device.resolver) devices() (ResourceSpec property) DeviceSpec (class in autodist.resource_spec) DeviceType (class in autodist.resource_spec) DiscardUnknownFields() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method)

E

ENV (class in autodist.const) env() (SSHConfig property) error() (in module autodist.utils.logging) ETHERNET (Connectivity attribute) extend_gradient_info() (GraphItem method) extend_gradient_info_by_names() (GraphItem method) extend_optimizer_info() (GraphItem method)

Extensions (AllReduceSynchronizer attribute) (GraphItem attribute) (GraphItem.GradTargetPairsEntry attribute) (GraphItem.Info attribute) (PSSynchronizer attribute) (Strategy attribute) (Strategy.GraphConfig attribute) (Strategy.Node attribute)

F

final_op (CollectiveOpsConfig attribute) FindInitializationErrors() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) from_string() (DeviceSpec class method)

fromkeys() (SSHConfigMap method) FromString() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) function() (AutoDist method)

G

gen_read_var_op() (in module autodist.kernel.common.variable_utils) gen_server() (in module autodist.utils.server_starter) get() (SSHConfigMap method) get_address_from_task() (Cluster method) (SSHCluster method) get_all_update_ops() (ProxyVariable method) get_all_variables() (GraphItem method) get_ancestors() (in module autodist.kernel.common.utils) get_collective_keys() (in module autodist.kernel.synchronization.collective_key) get_colocation_op() (GraphItem method) get_consumers() (in module autodist.kernel.common.utils) get_control_consumers() (in module autodist.kernel.common.utils) get_default_autodist() (in module autodist.autodist) get_default_graph_item() (in module autodist.graph_item) get_default_run_options() (in module autodist.runner) get_default_session_config() (in module autodist.runner) get_group_key() (CollectiveKey method) get_index_from_tensor_name() (in module autodist.kernel.common.utils) get_instance_key() (CollectiveKey method) get_local_address() (Cluster method) (SSHCluster method) get_local_session_target() (Cluster method) (SSHCluster method) get_local_worker_task_index() (Cluster method) (SSHCluster method) get_logger() (in module autodist.utils.logging) get_num_shards() (PartitionedAR static method) (PartitionedPS static method) (UnevenPartitionedPS static method)

get_num_shards_and_axis() (RandomAxisPartitionAR static method) get_op_name() (in module autodist.kernel.common.utils) get_ops_in_graph() (GraphItem method) get_read_var_ops() (in module autodist.kernel.common.variable_utils) get_read_var_tensor() (in module autodist.kernel.common.variable_utils) get_trainable_variables() (GraphItem method) get_verbosity() (in module autodist.utils.logging) GPU (DeviceType attribute) gpu_devices() (ResourceSpec property) GPU_TO_GPU_LOCAL (Connectivity attribute) GPU_TO_GPU_REMOTE (Connectivity attribute) grad_target_name_pairs() (GraphItem property) grad_target_pairs (GraphItem attribute) grad_target_pairs() (GraphItem property) graph() (GraphItem property) graph_config (Strategy attribute) graph_config() (Strategy property) graph_def (GraphItem attribute) GraphItem (class in autodist.graph_item) (class in autodist.proto.graphitem_pb2) GraphItem.GradTargetPairsEntry (class in autodist.proto.graphitem_pb2) GraphItem.Info (class in autodist.proto.graphitem_pb2) GraphTransformer (class in autodist.kernel.graph_transformer) group (AllReduceSynchronizer attribute) group_key (CollectiveOpsConfig attribute) group_size (CollectiveOpsConfig attribute)

H

HasExtension() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) HasField() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method)

HorovodCompressor (AllReduceSynchronizer attribute) (class in autodist.kernel.synchronization.compressor) HorovodCompressorEF (AllReduceSynchronizer attribute) (class in autodist.kernel.synchronization.compressor)

I

id (Strategy attribute) id() (Strategy property) in_graph_apply() (AllReduceSynchronizer method) (PSSynchronizer method) (Synchronizer method) index() (SSHConfig method) Info (class in autodist.graph_item) info (GraphItem attribute) info() (in module autodist.utils.logging) initializers() (Info property) instance_key (CollectiveOpsConfig attribute) is_built() (AutoDist method) is_chief() (Cluster method) (SSHCluster method)

is_local_address() (in module autodist.utils.network) is_loopback_address() (in module autodist.utils.network) is_read_var_op() (in module autodist.kernel.common.variable_utils) IsInitialized() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) items() (SSHConfigMap method)

J

join() (Coordinator method)

K

Kernel (class in autodist.kernel.kernel) key (GraphItem.GradTargetPairsEntry attribute)

key_file() (SSHConfig property) keys() (SSHConfigMap method)

L

launch_clients() (Coordinator method) ListFields() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method)

local_replication (PSSynchronizer attribute) log() (in module autodist.utils.logging) log_graph() (in module autodist.utils.visualization_util)

M

merge_op (CollectiveOpsConfig attribute) MergeFrom() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) MergeFromString() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) module autodist autodist.autodist autodist.checkpoint autodist.checkpoint.saved_model_builder autodist.checkpoint.saver autodist.cluster autodist.const autodist.coordinator autodist.graph_item autodist.kernel autodist.kernel.common autodist.kernel.common.op_info autodist.kernel.common.proxy_variable autodist.kernel.common.utils autodist.kernel.common.variable_utils autodist.kernel.device autodist.kernel.device.resolver autodist.kernel.graph_transformer autodist.kernel.kernel autodist.kernel.partitioner autodist.kernel.replicator autodist.kernel.synchronization autodist.kernel.synchronization.all_reduce_synchronizer autodist.kernel.synchronization.collective_key autodist.kernel.synchronization.compressor autodist.kernel.synchronization.ps_synchronizer autodist.kernel.synchronization.synchronizer autodist.patch autodist.proto autodist.proto.graphitem_pb2 autodist.proto.strategy_pb2 autodist.proto.synchronizers_pb2 autodist.remapper autodist.resource_spec autodist.runner autodist.simulator autodist.strategy autodist.strategy.all_reduce_strategy autodist.strategy.base autodist.strategy.parallax_strategy autodist.strategy.partitioned_all_reduce_strategy autodist.strategy.partitioned_ps_strategy autodist.strategy.ps_lb_strategy autodist.strategy.ps_strategy autodist.strategy.random_axis_partition_all_reduce_strategy autodist.strategy.uneven_partition_ps_strategy autodist.utils autodist.utils.logging autodist.utils.network autodist.utils.server_starter autodist.utils.visualization_util

N

name_string() (DeviceSpec method) NCCL (AllReduceSynchronizer attribute) network_bandwidth() (ResourceSpec property) node_config (Strategy attribute) node_config() (Strategy property) node_cpu_devices() (ResourceSpec property) node_gpu_devices() (ResourceSpec property)

nodes() (ResourceSpec property) NoneCompressor (AllReduceSynchronizer attribute) (class in autodist.kernel.synchronization.compressor) num_cpus() (ResourceSpec property) num_gpus() (ResourceSpec property) num_local_replicas() (GraphTransformer property) num_shards() (PartitionerConfig property)

P

Parallax (class in autodist.strategy.parallax_strategy) parse_name_scope() (in module autodist.kernel.common.utils) parse_optimizer_scope() (in module autodist.kernel.common.utils) ParseFromString() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) part_config (Strategy.Node attribute) partition_list() (PartitionerConfig property) partition_str() (PartitionerConfig property) PartitionedAR (class in autodist.strategy.partitioned_all_reduce_strategy) PartitionedPS (class in autodist.strategy.partitioned_ps_strategy) partitioner (Strategy.Node attribute) PartitionerConfig (class in autodist.kernel.partitioner) patch_keras() (PatchTensorFlow static method)

patch_optimizers() (PatchTensorFlow static method) patch_var_reading() (PatchTensorFlow static method) PatchTensorFlow (class in autodist.patch) path (Strategy attribute) path() (Strategy property) pkey() (SSHConfig property) pop() (SSHConfigMap method) pop_gradient_info() (GraphItem method) pop_variable() (Info method) popitem() (SSHConfigMap method) port() (SSHConfig property) prepare() (GraphItem method) ProxyVariable (class in autodist.kernel.common.proxy_variable) PS (class in autodist.strategy.ps_strategy) PSGradientTaskAssigner (class in autodist.kernel.synchronization.ps_synchronizer) PSLoadBalancing (class in autodist.strategy.ps_lb_strategy) PSSynchronizer (class in autodist.kernel.synchronization.ps_synchronizer) (class in autodist.proto.synchronizers_pb2) (Strategy.Node attribute) python_venv() (SSHConfig property)

R

RandomAxisPartitionAR (class in autodist.strategy.random_axis_partition_all_reduce_strategy) reduce() (Compressor method) (CompressorEF method) (HorovodCompressor method) (HorovodCompressorEF method) (NoneCompressor method) reduction_destination (PSSynchronizer attribute) RegisterExtension() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) remap_callable_options() (Remapper method) Remapper (class in autodist.remapper) remote_copy() (Cluster method) (SSHCluster method)

remote_exec() (Cluster method) (SSHCluster method) remote_file_write() (Cluster method) (SSHCluster method) remote_pre_start_tf_server() (Cluster method) (SSHCluster method) remove_from_control_consumers() (in module autodist.kernel.common.utils) replica_prefix() (in module autodist.kernel.common.utils) replicas (Strategy.GraphConfig attribute) replicate() (Replicator method) Replicator (class in autodist.kernel.replicator) reset_optimize() (GraphItem method) resolve_to_device_spec() (DeviceResolver method) resolve_to_device_str() (DeviceResolver method) ResourceSpec (class in autodist.resource_spec) restore() (Saver method) RING (AllReduceSynchronizer attribute) run() (WrappedSession method)

S

SAME (Connectivity attribute) save() (Saver method) SavedModelBuilder (class in autodist.checkpoint.saved_model_builder) Saver (class in autodist.checkpoint.saver) savers (GraphItem.Info attribute) scope() (AutoDist method) serialize() (GraphItem method) (PartitionerConfig method) (Strategy method) SerializePartialToString() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) SerializeToString() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) set_default_autodist() (in module autodist.autodist) set_device_resolver() (StrategyCompiler method) set_optimize() (GraphItem method) set_verbosity() (in module autodist.utils.logging) setdefault() (SSHConfigMap method)

SetInParent() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) SHARED_TASK_ID (PSGradientTaskAssigner attribute) Spec (AllReduceSynchronizer attribute) spec (AllReduceSynchronizer attribute) ssh_config_map() (ResourceSpec property) ssh_group() (ResourceSpec property) SSHCluster (class in autodist.cluster) SSHConfig (class in autodist.resource_spec) SSHConfigMap (class in autodist.resource_spec) staleness (PSSynchronizer attribute) start() (Cluster method) (SSHCluster method) start_server() (in module autodist.utils.server_starter) Strategy (class in autodist.proto.strategy_pb2) (class in autodist.strategy.base) Strategy.GraphConfig (class in autodist.proto.strategy_pb2) Strategy.Node (class in autodist.proto.strategy_pb2) StrategyBuilder (class in autodist.strategy.base) StrategyCompiler (class in autodist.strategy.base) strip_replica_prefix() (in module autodist.kernel.common.utils) sync (PSSynchronizer attribute) Synchronizer (class in autodist.kernel.synchronization.synchronizer) SYS_DATA_PATH (ENV attribute) SYS_RESOURCE_PATH (ENV attribute)

T

table_initializers (GraphItem.Info attribute) terminate() (Cluster method) (SSHCluster method)

trainable_var_op_to_var() (GraphItem property) trainable_variables() (Info property) transform() (GraphTransformer method) traverse() (in module autodist.kernel.common.utils)

U

UnevenPartitionedPS (class in autodist.strategy.uneven_partition_ps_strategy) UnknownFields() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method) unpatch_keras() (PatchTensorFlow static method)

unpatch_var_reading() (PatchTensorFlow static method) untrainable_variables() (Info property) update() (SSHConfigMap method) update_colocation_group() (in module autodist.kernel.common.utils) (ProxyVariable method) update_consumers() (in module autodist.kernel.common.utils) update_control_consumers() (in module autodist.kernel.common.utils) update_savers() (Info method) update_table_initializers() (Info method) update_variables() (Info method) username() (SSHConfig property)

V

val() (ENV property) value (GraphItem.GradTargetPairsEntry attribute) values() (SSHConfigMap method) var_name (Strategy.Node attribute)

var_op_name_to_grad_info() (GraphItem property) var_op_name_to_grad_info_v2() (GraphItem property) VariablePartitioner (class in autodist.kernel.partitioner) variables (GraphItem.Info attribute)

W

warning() (in module autodist.utils.logging) WhichOneof() (AllReduceSynchronizer method) (GraphItem method) (GraphItem.GradTargetPairsEntry method) (GraphItem.Info method) (PSSynchronizer method) (Strategy method) (Strategy.GraphConfig method) (Strategy.Node method)

wrap_optimizer_apply_gradient() (in module autodist.graph_item) wrap_optimizer_init() (in module autodist.graph_item) WrappedSession (class in autodist.runner)