B λd@sdZddlZddlZddlZddlmZmZmZmZmZm Z m Z m Z ddl m Z ddlZddlmZddlmZddlmZddlmZdd lmZddlmZdd lmZdd lmZdd l m!Z!dd l m"Z"ddl m#Z#ddl m$Z$ddl m%Z%ddl m&Z'ddl m(Z(ddl m)Z)ddl m*Z*ddl m+Z+ddl m,Z,ddl m-Z-ddl.m/Z/ddl.m0Z0ddl.m1Z1ddl.m2Z2ddl.m3Z3ddl4m5Z6ddl4m7Z7ddl4m8Z8dd l4m9Z9dd!l:m;Z;dd"ldd#l?m@Z@dd$l?mAZAdd%l?mBZBdd&l?mCZCdd'l?mDZDdd(lEmFZFe,Gd)eHd*gZIeHd+d,d-d.d/d0d1d2d3g ZJeHd*d4gZKd5ZLd6ZMd7ZNd8ZOd9ZPd:ZQe9j=Z=ee e d;dgd?deejSee eTeeTe>jUdAdBdCZVdejSee e,jWdDdEdFZXeFdGgd?dee e,jWd;dHdIZYe eefdJdKdLZZeeTdMdNdOZ[ee6j\dJdPdQZ]e!j^e,jWeBj_eBj_eTdRdSdTZ`GdUdVdVe0jaZbGdWdXdXe0jcZdeFdYgd?edZefed[d\d]ZeeFd^gd?Gd_d`d`ejfZgeFdaGdbdcdcehdcdddegZieFdfgd?eCjjdedZefeeee>jUee7jkee6j\ee eeeegeeieedg dhdiZldjdkZmee>jUeee-jnege eeeeejofdldmdnZpddodpZqdedZefeeee>jUee7jkee6j\ee eTeeeegeeieee>jUdq drdsZreeTe eee>jUee,jWeefdtdudvZseeTe eee>jUee,jWeefdtdwdxZtdedZefeeeee>jUeueeeue eTeeTfeeeuee7jkee6j\ee eeie e,jWee>jUfdz d{d|ZveFd}gd?eCjjdedZefeee>jUeueeeue eTeeTfeeeuee7jkee6j\ee eeiee>jUdz d~dZweFdgd?eCjjdedZefeeeee>jUeuee7jkee6j\ee eeidddZxeFdgd?eCjjdedZefeeeee>jUee7jkee6j\ee eeiedddZyeHddddd4ddgZzeTdJddZ{Gddde0jaZ|e,j}dddZ~dedZefeee>jUee7jkee6j\ee ee>jUdddZe,j}dddZdS)z Library of TPU helper functions.N)AnyCallableIterableListOptionalTextTupleUnion)logging)xla)attr_value_pb2)dynamic_padding_pb2)tpu_embedding_configuration_pb2)tf2) device_util)distribution_strategy_context)auto_control_deps) c_api_util)composite_tensor)config) constant_op)device)dtypes)errors) func_graph)function)ops) tensor_shape) array_ops)control_flow_ops)math_ops)variable_scope) variables)device_assignment)tpu_feed) tpu_function) tpu_name_util)tpu_ops)core)compat)nest)object_identity)traceback_utils)variable_utils) tf_exportTPUReplicatedInputZ PlaceholderZ AudioSummaryZAudioSummaryV2ZHistogramSummaryZ ImageSummaryZ MergeSummaryZPrintZ ScalarSummaryZ TensorSummaryZTensorSummaryV2Z VarHandleOpZ_tpu_replicateZ_post_device_rewriteZ_tpu_compilation_status_xla_outside_compilationZ_pivot_for_cluster)jobreturncCs|dkr dSd|SdS)z;Returns the device name for the TPU_SYSTEM device of `job`.Nz/device:TPU_SYSTEM:0z/job:%s/device:TPU_SYSTEM:0)r2r4r4K/var/www/html/venv/lib/python3.7/site-packages/tensorflow/python/tpu/tpu.py_tpu_system_device_namegsr6ztpu.initialize_system)Zv1T)embedding_configr2 compilation_failure_closes_chipstpu_cancellation_closes_chipsr3c Cs|dkr dn|}d}|dk r.|r*d}nd}tt|ftj||d}|dkrX|St|gtj|d}WdQRXt|gtj |dd SQRXWdQRXdS) aInitializes a distributed TPU system for use with TensorFlow. Args: embedding_config: If not None, a `TPUEmbeddingConfiguration` proto describing the desired configuration of the hardware embedding lookup tables. If embedding_config is None, no hardware embeddings can be used. job: The job (the XXX in TensorFlow device specification /job:XXX) that contains the TPU devices that will be initialized. If job=None it is assumed there is only one job in the TensorFlow flock, and an error will be returned if this assumption does not hold. compilation_failure_closes_chips: Set the configuration whether we want to close TPU chips when there is a compilation failure. tpu_cancellation_closes_chips: Set the configuration whether we want to close TPU chips when a TPU execution is cancelled. If the value is None, the behavior will be determined by the command line flag `tpu_cancellation_closes_chips` for the TPU worker. WARNING: this argument only applies to TFRT TPU runtime. Returns: A serialized `TopologyProto` that describes the TPU system. Note: the topology must be evaluated using `Session.run` before it can be used. Nr)r8r9)rZtpu_init_identity)name) SerializeToStringrrr6r'Zconfigure_distributed_tpucontrol_dependenciesconfigure_tpu_embeddingridentity)r7r2r8r9 config_stringZ"tpu_cancellation_closes_chips_enumtopologyZembedding_initr4r4r5initialize_systemos" rD)r7r2r3c Cs.|}tt|tj|dSQRXdS)aVInitializes a distributed TPU Embedding system for use with TensorFlow. The following two are equivalent: 1. initialize_system() with embedding_config. 2. initialize_system() without embedding_config, then initialize_system_for_tpu_embedding(). initialize_system() should not be called with embedding_config if initialize_system_for_tpu_embedding() is meant to be called later. Args: embedding_config: a `TPUEmbeddingConfiguration` proto describing the desired configuration of the hardware embedding lookup tables. job: The job (the XXX in TensorFlow device specification /job:XXX) that contains the TPU devices that will be initialized. If job=None it is assumed there is only one job in the TensorFlow flock, and an error will be returned if this assumption does not hold. Returns: A no-op. )rN)r>rrr6r'r@)r7r2rBr4r4r5#initialize_system_for_tpu_embeddingsrEztpu.shutdown_systemc Cs&tt|t}WdQRX|S)aQShuts down a running a distributed TPU system. Args: job: The job (the XXX in TensorFlow device specification /job:XXX) that contains the TPU devices that will be shutdown. If job=None it is assumed there is only one job in the TensorFlow flock, and an error will be returned if this assumption does not hold. N)rrr6r'shutdown_distributed_tpu)r2rFr4r4r5shutdown_systems rG)r3cCs\t}xF|dk rN|}x$|dk r>t|tr6||fS|j}qWt|dd}q WtddS)z9Returns the TPUReplicateContext and its associated graph.N outer_graphziget_replicated_var_handle() called without TPUReplicateContext. This shouldn't happen. Please file a bug.)rget_default_graph_get_control_flow_context isinstanceTPUReplicateContext outer_contextgetattr ValueError)graphZcontext_r4r4r5 _enclosing_tpu_context_and_graphs    rQ)strategyr3cCs&dd}|j}||p$tt||jS)NcSs |jdS)NZ TPUStrategy)__name__ startswith)kr4r4r5z!is_tpu_strategy..) __class__anymap __bases__)rRZ is_tpu_stratZclzr4r4r5is_tpu_strategysr\cCs(ts dSt}t|s dS|jjS)N)rZ has_strategyZ get_strategyr\extendedZ_device_assignment)rRr4r4r5 _enclosing_tpu_device_assignments r^)opresource_readsresource_writesr3cCsF|jdkr*|s|r&||dSdSdd}t||pB||S)zGReplaces TPUReplicatedInput outputs with its inputs in resource_inputs.r/TFcSsdg}g}x0|D](}|jjdkr||||jjqWx|D]}||q@W|||pb|S)zEReplaces handles in `resource_inputs` with their unreplicated inputs.r/)r_typeappendextendinputsdiscardupdate)Zresource_inputsZ to_removeZto_addresourcetr4r4r5#replace_with_unreplicated_resources s     zJtpu_replicated_input_resolver..replace_with_unreplicated_resources)rbclearbool)r_r`rarjr4r4r5tpu_replicated_input_resolvers  rmcsZeZdZdZeeejdfdd Zd.eee e e j e e jfeee j dddZd d d d Zejed ddZejed ddZd/eedddZddZd d fdd Ze ed ddZejee eje ejfdddZejd dddZe j e j d d!d"Zejd#d$d%Zed&d'Zed(d)Z ejd d*d+Z!d,d-Z"Z#S)0rLahA `ControlFlowContext` for nodes inside a TPU computation. The primary role of `TPUReplicateContext` is to mark operators inside a tpu.replicate() computation with the attribute "_tpu_replicate=XYZ", where XYZ is a unique name. We use a `ControlFlowContext` to perform the annotation since it integrates with Tensorflow constructs like ResourceVariables. For example, if a `ResourceVariable` is constructed inside a tpu.replicate() block, the `ResourceVariable` implementation can use `with ops.control_dependencies(None)` to build the variable's definition outside the replicated computation. )r= num_replicaspivotcstt|||_d|_d|_d|_d|_d|_d|_ g|_ g|_ ||_ t ||_ttj|jd|_g|_||_i|_dS)aBuilds a new TPUReplicateContext. Args: name: a unique name for the context, used to populate the `_tpu_replicate` attribute. num_replicas: an integer that gives the number of replicas for the computation. pivot: a pivot node. Nodes in the TPUReplicateContext that do not have any inputs will have a control dependency on the pivot node. This ensures that nodes are correctly included in any enclosing control flow contexts. Nr)s)superrL__init__ _num_replicas_outer_device_function_stack_oc_dev_fn_stack_outside_compilation_cluster_outside_compilation_v2_context_outside_compilation_counter_in_gradient_colocation_gradient_colocation_stack_host_compute_core_namer)as_bytesZ_name_as_bytesrZScopedTFBufferr AttrValuer>_tpu_relicate_attr_buf_unsupported_ops_pivot_replicated_vars)selfr=rnro)rXr4r5rr)s"  zTPUReplicateContext.__init__F)r= handle_idvars_ is_mirrored is_packedr3c Cs8t}|j|}|dk r|S|dk r|stj|djj}dd|D} g} xht|j D]T} xNt|j D]2} t |j | | |d} | | krl| | | PqlWtd| q\Wn|} t\}}|Xt| dtjrdd| D} |}||jtj| |d ||d }||WdQRX||j|<|S) aReturns a variable handle for replicated TPU variable 'var'. This is a method used by an experimental replicated variable implementation and is not intended as a public API. Args: name: The common name of the variable. handle_id: Unique ID of the variable handle, used as the cache key. vars_: The replicated TPU variables or handles. is_mirrored: Whether the variables are mirrored, which guarantees the values in each replica are always the same. is_packed: Whether the replicated variables are packed into one variable. Returns: The handle of the TPU replicated input node. NrcSsi|]}|t|jqSr4)r canonicalizer).0vr4r4r5 kszATPUReplicateContext.get_replicated_var_handle..)replica logical_corer2zSFailed to find a variable on any device in replica {} for current device assignmentcSsg|] }|jqSr4)handle)rrr4r4r5 szATPUReplicateContext.get_replicated_var_handle..z/handle)r=Zis_mirrored_variabler)r^rgetpydev DeviceSpec from_stringrr2rangernnum_cores_per_replicarrZ tpu_devicercrOformatrQZ as_defaultrKr"VariablerJZ_set_control_flow_contextrMr'tpu_replicated_input)rr=rrrrr#rZjob_nameZdevices_to_varsZreplicated_varsZ replica_idrr_rPZ saved_contextr4r4r5get_replicated_var_handleHsB      z-TPUReplicateContext.get_replicated_var_handleN)r3cCsb|jr^ddd|jdtD}tdt|j|t|jtkr^tdt|jtdS)N css|]}d|j|jfVqdS)z %s (%s)N)rbr=)rr_r4r4r5 szDTPUReplicateContext.report_unsupported_operations..z$%d unsupported operations found: %sz... and %d more)rjoin_MAX_WARNING_LINESr warninglen)rZop_strr4r4r5report_unsupported_operationss z1TPUReplicateContext.report_unsupported_operations)r_ gradient_uidcCs|dk rtjdkrry|td}Wntk r>dSX|d}|dd|}t||_ |j dS|j ||j sy^|td}|jrtd|dkrtd||_|d}|dd|}|j|dWntk rYnXdS)Nascii.rz>Cannot nest gradient colocation operations outside compilationZ__unsupported__z;No gradient_uid calling gradient within outside_compilation)cluster)rrI_control_flow_contextZget_attr_OUTSIDE_COMPILATION_ATTRdecoderOsplitOutsideCompilationV2ContextrwEnterrzrcrvryNotImplementedError_EnterOutsideCompilationScope)rr_rZ outside_attrpartsrr4r4r5EnterGradientColocations8    z+TPUReplicateContext.EnterGradientColocationcCs|dk rtjdkr(|jdks$tdS|jdk rF|jd|_dS|jsdt|j |d|j |j }||kr||j krd|_ | nt|j |d|d|j dS)Nz>Badly nested gradient colocation: empty stack when popping Op z+Badly nested gradient colocation, expected z, got )rrIrrwAssertionErrorExitrzrZ InternalErrornode_defr=popry_ExitOutsideCompilationScope)rr_rZlast_opr4r4r5ExitGradientColocations(     z*TPUReplicateContext.ExitGradientColocation)rcCsGdddt}|jrtd|r*||_nt|j|_|jd7_t}|}||tj |j }|j dkr|j dk r|j|jdt|j |j|_|j|_dS)Nc@s@eZdZdZddZeddZeddZdd Zd d Z d S) zATPUReplicateContext._EnterOutsideCompilationScope..FakeOpzA helper class to determine the current device. Supports only the type and device set/get methods needed to run the graph's _apply_device_function method. cSs d|_dS)Nr:)_device)rr4r4r5rrszJTPUReplicateContext._EnterOutsideCompilationScope..FakeOp.__init__cSsdS)NFakeOpr4)rr4r4r5rbszFTPUReplicateContext._EnterOutsideCompilationScope..FakeOp.typecSs|jS)N)r)rr4r4r5rszHTPUReplicateContext._EnterOutsideCompilationScope..FakeOp.devicecSs"t|tjr||_n||_dS)N)rKrrZ to_stringr)rrr4r4r5 _set_devices  zMTPUReplicateContext._EnterOutsideCompilationScope..FakeOp._set_devicecSs ||_dS)N)r)rZ device_strr4r4r5_set_device_from_string szYTPUReplicateContext._EnterOutsideCompilationScope..FakeOp._set_device_from_stringN) rS __module__ __qualname____doc__rrpropertyrbrrrr4r4r4r5rs   rz(Cannot nest outside_compilation clustersr;ZTPU_REPLICATED_CORE:)objectrvrstrrxrrIZ_apply_device_functionsrrrrZ device_typeZ device_indexr{rc_device_function_stackrurt)rrrrPZfake_oprr4r4r5rs"    z1TPUReplicateContext._EnterOutsideCompilationScopecCs(|jstdd|_t}|j|_dS)Nz=Attempted to exit outside_compilation scope when not in scope)rvrOrrIrur)rrPr4r4r5r"s z0TPUReplicateContext._ExitOutsideCompilationScopecs,|jst}|j|_tt|dS)N)rtrrIrcopyrqrLr)rrP)rXr4r5r*s zTPUReplicateContext.EntercCs|jS)N)r{)rr4r4r5HostComputeCore4sz#TPUReplicateContext.HostComputeCore)r_r3cCszg}g}xV|jD]L}d}|}x |dk r@||kr8d}P|j}q"W|rR||q||qW|||||fS)z2Remove any external control dependency on this op.FNT)Zcontrol_inputsrJ_outer_contextrcZ_remove_all_control_inputs_add_control_inputs)rr_internal_control_inputsexternal_control_inputsxZis_internal_opZctxtr4r4r5_RemoveExternalControlEdges7s      z/TPUReplicateContext._RemoveExternalControlEdgesc Cs|jtkrtd|j|j|jtkr2|j|tdd|j DrXt d|jdt |j j krd|j j krtd|d|t |jj|jr|ttjt|jd|jd ks|js|j||j|||\}}|j s|sJ||nDxBt t!|j D]0}|j |}|"|}||k r|#||qW|rt$%d$|&d d |D}|'WdQRX|(|d d |j)D}|}x"|dk r|j*+||j,}qW|j,r|j,-|dS) NzlOperation of type %s (%s) is not supported on the TPU. Execution will fail if this op is used in the graph. css|]}|jjVqdS)N)dtypeZ _is_ref_dtype)rrr4r4r5r[sz,TPUReplicateContext.AddOp..zQNon-resource Variables are not supported inside TPU computations (operator name: )Z_clonedz)TPU computations cannot be nested on op ()rpr;cSs$g|]}|jrt|jdjqS)r)outputsrrAr_)rrr4r4r5rsz-TPUReplicateContext.AddOp..cSsg|] }|jqSr4)r=)rrr4r4r5rs).rb_DENYLISTED_OPSr errorr=_UNSUPPORTED_OPSrrcrYrer_TPU_REPLICATE_ATTRrattrrOZ_set_attr_with_bufrbufferrv _set_attrrr r~r)r}rsrPZprevent_feedingZprevent_fetchingrZ_add_control_inputGetControlPivotrrAddValueZ _update_inputrr?rrrr_valuesrgr AddInnerOp) rr_rrindexrZreal_xZ output_namescontextr4r4r5AddOpQsX                 zTPUReplicateContext.AddOp)valr3cCsv|js |S|j|jkr4|j|j}|dkr0|S|S|}|j|j|jrf|j|}|j|j||j|j<|S)zCAdd `val` to the current context and its outer context recursively.N)rr=rZ_external_valuesraddr)rrresultr4r4r5rs   zTPUReplicateContext.AddValue)r_cCs |||jr|j|dS)N)rrr)rr_r4r4r5rs zTPUReplicateContext.AddInnerOpcCsdS)Nr4)rr4r4r5 grad_stateszTPUReplicateContext.grad_statecCs|r|jSdS)z0Forwards to the enclosing while context, if any.F)ZGetWhileContext back_prop)rr4r4r5rs zTPUReplicateContext.back_propcCs|jS)N)r)rr4r4r5rsz#TPUReplicateContext.GetControlPivotcCsdS)NTr4)rr4r4r5RequiresUniqueFunctionRetracingsz3TPUReplicateContext.RequiresUniqueFunctionRetracing)FF)N)$rSrrrrintr Operationrrr r core_typesTensorr"rrlrrrrrrrrrrrrrrrrrrr __classcell__r4r4)rXr5rLs( $0H 6/ "N  rLc@sLeZdZdZedddZejddddZejddd d Z d d d Z dS)rzThe context for outside compilation in Tensorflow 2.0. Every op added in this context will be assigned an _xla_outside_compilation attribute. )r=cCstj|||_dS)N)rControlFlowContextrrr|)rr=r4r4r5rrs z$OutsideCompilationV2Context.__init__N)r_r3cCs2|jr|j||dtjt|jddS)Nr1)rp)rrrr r~r)r}r|)rr_r4r4r5rs z!OutsideCompilationV2Context.AddOpcCs2|jr|j||dtjt|jddS)Nr1)rp)rrrr r~r)r}r|)rr_r4r4r5rs z&OutsideCompilationV2Context.AddInnerOpcCstdS)N)r)rZ context_defZ export_scoper4r4r5to_control_flow_context_defsz7OutsideCompilationV2Context.to_control_flow_context_def)N) rSrrrrrrrrrrrr4r4r4r5rs rztpu.outside_compilation.) computationr3c Os"|dkr gn|}t}t|tjryt\}}Wn$tk rVtd|||SXt |j }|j d|_ t |}| |dkrgn|}|||}| |S|} | } x | rt| tr| | j} qW|||}|} | | k rtd| } x$| rt| tr| | j} qW|S)a Builds part of a computation outside any current TPU replicate scope. `tf.tpu.outside_compilation()` is used to run ops in `computation` on CPU instead of running on TPU. For example, users can run ops that are not supported on TPU's (e.g. tf.summary.write()) by explicitly placing those ops on CPU's. Below usage of outside compilation will place ops in `computation_with_string_ops` on CPU. Example usage: ```python def computation_with_string_ops(x): # strings types are not supported on TPU's and below ops must # run on CPU instead. output = tf.strings.format('1{}', x) return tf.strings.to_number(output) def tpu_computation(): # Expected output is 11. output = tf.tpu.outside_compilation(computation_with_string_ops, 1) ``` Outside compilation should be called inside TPUReplicateContext. That is, `tf.tpu.outside_compilation()` should be called inside a function that is passed to `tpu.split_compile_and_replicate()` -- this is implied when outside compilation is invoked inside a function passed to TPUStrategy `run()`. If invoked outside of TPUReplicateContext, then this simply returns the result of `computation`, and therefore, would be a no-op. Note that outside compilation is different from `tf.distribute.experimental.TPUStrategy.merge_call()` as logic in outside compilation is replicated and executed separately for each replica. On the other hand, `merge_call()` requires a `merge_fn` to aggregate the inputs from different replicas and is executed only once. For variables placed in TPU device, which includes variables created inside TPUStrategy scope, outside compilation logic must not include variable read/write. For variables placed on host, which is the case when variables created via TPUEstimator, variable read/write is only allowed if the variable is not accessed by any other ops in the TPU computation. Variable read/write from outside compilation cluster is not visible from TPU computation and vice versa. Therefore, if outside compilation logic contains such host variables read/write ops and if the variables are accessed by TPU computation as well, then this may lead to deadlock. Internally, `tf.tpu.outside_compilation()` adds outside compilation attributes to all ops in `computation`. During later graph pass, these ops with outside compilation attribute is extracted out and replicated into a host-side graph. Inputs to this extract host-side graph is sent from TPU computation graph to host graph via a pair of XlaSendToHost and XlaRecvFromHost ops. Note that using `tf.tpu.outside_compilation()` may result in tensor transfer between TPU and CPU, leading to non-trivial performance impact. Args: computation: A Python function that builds the computation to place on the host. *args: the positional arguments for the computation. **kwargs: the keyword arguments for the computation. Returns: The Tensors returned by computation. NzOutside compilation attempted outside TPUReplicateContext scope. As no enclosing TPUReplicateContext can be found, returning the result of `computation` as is.r;zYControl-flow context cannot be different at start and end of an outside_compilation scope)rrIrKr FuncGraphrQrOr rrrxrrrrJrLrrMrr) rargskwargsrPZ tpu_contextrZoutside_compilation_nameZoutside_compilation_contextretvalZinitial_contextrZ final_contextr4r4r5outside_compilationsFC          rztpu.PaddingSpecc@seZdZdZdZdZdS) PaddingSpecz:Represents the type of padding policies for tpu.replicate.rr;N)rSrrrZAUTO POWER_OF_TWOr4r4r4r5rlsrztpu.XLAOptionscs"eZdZdZdfdd ZZS) XLAOptionsa{XLA compilation options. Attributes: use_spmd_for_xla_partitioning: Boolean. Whether to use XLA's SPMD partitioner instead of MPMD partitioner when compiler partitioning is requested. enable_xla_dynamic_padder: Boolean. Whether to enable XLA dynamic padder infrastructure to handle dynamic shapes inputs inside XLA. True by default. Disabling this may cause correctness issues with dynamic shapes inputs, as XLA will just assume the inputs are with padded shapes. However users can optionally set it to False to improve device time if masking is already handled in the user side. Tcstt||||S)N)rqr__new__)clsuse_spmd_for_xla_partitioningenable_xla_dynamic_padder)rXr4r5rszXLAOptions.__new__)TT)rSrrrrrr4r4)rXr5rvsrrrz tpu.replicate) rre infeed_queuer#r=maximum_shapes padding_spec xla_optionsr3c Cst||||||||ddS)aBuilds a graph operator that runs a replicated TPU computation. Example for the basic usage that `inputs` has static shape: ```python def computation(x): x = x + 1 return tf.math.reduce_mean(x) x = tf.convert_to_tensor([1., 2., 3.]) y = tf.convert_to_tensor([4., 5., 6.]) tf.compat.v1.tpu.replicate(computation, inputs=[[x], [y]]) ``` If the `inputs` has dynamic shapes and you would like to automatically bucketize the inputs to avoid XLA recompilation. See the advanced example below: ```python def computation(x): x = x + 1 return tf.math.reduce_mean(x) # Assume input tensors in two replicas `x` and `y` both have dynamic shape # ([None, 2]). tf.compat.v1.tpu.replicate( computation, inputs=[x, y], maximum_shapes=[tf.TensorShape([None, None])], padding_spec=tf.compat.v1.tpu.PaddingSpec.POWER_OF_TWO) ``` Args: computation: A Python function that builds the computation to replicate. inputs: A list of lists of input tensors or `None` (equivalent to `[[]]`), indexed by `[replica_num][input_num]`. All replicas must have the same number of inputs. Each input can be a nested structure containing values that are convertible to tensors. Note that passing an N-dimension list of compatible values will result in a N-dimension list of scalar tensors rather than a single Rank-N tensors. If you need different behavior, convert part of inputs to tensors with `tf.convert_to_tensor`. infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to computation. device_assignment: If not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. Uses a default device assignment if `None`. The `DeviceAssignment` may be omitted if each replica of the computation uses only one core, and there is either only one replica, or the number of replicas is equal to the number of cores in the TPU system. name: (Deprecated) Does nothing. maximum_shapes: A nested structure of tf.TensorShape representing the shape to which the respective component of each input element in each replica should be padded. Any unknown dimensions (e.g. tf.compat.v1.Dimension(None) in a tf.TensorShape or -1 in a tensor-like object) will be padded to the maximum size of that dimension over all replicas. The structure of `maximum_shapes` needs to be the same as `inputs[0]`. padding_spec: An enum specified by `tpu.PaddingSpec`. This describes the padding policy when the `inputs` to `tpu.replicate` is dynamic. One usage is to enable automatic bucketizing on the inputs by setting the value to `tpu.PaddingSpec.POWER_OF_TWO`, which can help to reduce the recompilation in the XLA side. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: A list of outputs, indexed by `[replica_num]` each output can be a nested structure same as what computation() returns with a few exceptions. Exceptions include: 1) None output: a NoOp would be returned which control-depends on computation. 2) Single value output: A tuple containing the value would be returned. 3) Operation-only outputs: a NoOp would be returned which control-depends on computation. TODO(b/121383831): Investigate into removing these special cases. Raises: ValueError: If all replicas do not have equal numbers of input tensors. ValueError: If the number of inputs per replica does not match the number of formal parameters to `computation`. ValueError: If the static `inputs` dimensions don't match with the values given in `maximum_shapes`. ValueError: If the structure of inputs per replica does not match the structure of `maximum_shapes`. )rrrr;)split_compile_and_replicate)rrerr#r=rrrr4r4r5 replicatesbrcCsRt|tj}t|t|d}t|}t|d|}t|tj}|S)zCeil input `x` to power of `n`.g?)r castrZfloat32logceilpowint32)rnZlognxrr4r4r5_ceil_to_pow_of_ns  r)re padded_shapesrr3c svg}g}g}xt|D]\}}xt|D]\}} |dkrp|g|| |tj| dtdnLx8t| D],\} } | dks| ||| krzd||| <qzWt| ||||<t } | j t t jdd||| q(WqWg} x&|D]}| tjt|ddqWg}g}g}xt|D]\}}|g|gt|d}xt|D]\}|||}} ||}t||r&|dk r&xvt| D]j\} } ||| r|dkr|d7}t}||_| |_||_||||t|| tjqWgxt|jD]\} } ||| rd }| jdk rjt| j|}n*t| || |}|tj krt!|d }d||| g}nddg}|q6W"rt#$t%dfd d fd d }n t&}|j t t jdd|||n||qlWq6Wt|}x$t'|D]} || (|| qRW||fS) aPad all input tensors given padded_shapes. The real shape tensors will be concatenated with the padded original inputs. Args: inputs: The original inputs. padded_shapes: A list of padded shapes for each input. If an entry is None, no padding is performed. padding_spec: An enum specified by `tpu.PaddingSpec`. This describes the padding policy when the `inputs` to `tf.tpu.replicate` is dynamic. One usage is to enable automatic bucketizing on the inputs by setting the value to `tpu.PaddingSpec.POWER_OF_TWO`, which can help to reduce the recompilation in the XLA side. Returns: The padded inputs and a PaddingMap list which maps the padded input dimension to the real shape argument index. rF)rNT)b)axisr;r<cs tS)N)rpadr4) input_tensorpaddingsr4r5rVvrWz _pad_all_input..csS)Nr4r4)rr4r5rVwrW)) enumerate get_shapeas_listrcnpZ full_likerlmaxrshaper_r_POST_DEVICE_REWRITE_ATTRr r~r Z reduce_maxstackrrYdynamic_padding PaddingMap arg_index shape_indexpadding_arg_indexrrrZdimsvaluemaximumrrrZis_fully_definedrZcondconstantrrrd)rerrZmaximum_static_shapesZ need_paddingZinput_shape_tensorsZcore_idxZinputs_per_coreidx input_shapeirpZreal_input_shaperZshapes_per_inputZ padded_inputsZ real_shapes padding_mapsZreal_shape_idxZinput_shape_tensorZ padded_shape padding_mapZminimum_dynamic_dim_sizeZ max_dim_sizepaddingZ padded_inputrnr4)rrr5_pad_all_inputs                 rcCs,t|tjr(ttj|dd}|f|S|S)aiFor an input, replaced the input by a tuple if the input is composite. If `maybe_composite` is not composite, return the parameter `non_composite_output` otherwise return a tuple which consists of the value of the parameter `composite_output` the same number of times as there are components of the composite tensor. This is useful for computing a mask when flattening nested data with `expand_composites=True`. For example ```python nest.flatten(data, expand_composites=True) ``` and ```python nest.flatten(nest.map( data, lambda x: _flatten_and_filter_composite(x, False, True))) ``` will have the same length and second will be True if the tensor in the first is derived from a expanding a composite tensor. Args: maybe_composite: A value to test for being a composite tensor. non_composite_output: The value to return when `maybe_composite` is not a composite. composite_output: the value to fill the output tuple with if `maybe_composite` is a composite. Returns: `non_composite_output` or a tuple with multiple copies of `composite_output`. T)expand_composites)rKrZCompositeTensorrr*flatten)Zmaybe_compositeZnon_composite_outputZcomposite_outputZnum_componentsr4r4r5_flatten_and_filter_composites&  r!) rrerr#r=use_tpurrrr3c 3 s~|dkrggn|}|pt}i} |dk rN|j|jd} |j| d<t| d<trlt dt |t st dt|tdd|Drt d d d |Dt|} | d krgSx&td | D]t|d |qWt|}dd |D} ttdd|d } gx"| D]} dd | Dq*Wdd d D}t|d }t|}xlt| D]`t||krtd|t|dd D}||krxtd||qxWt|||}|dk r\|dkr(t d|ddd |d Dd|n4t d|ddd |d Ddd|jd|d}|r|rttdtj|d |dd td!d tt|d t|D}d"d |D}tjd |dd }t|||\}|rd#}t d$|d t|d%d&| d%<|j | d'<t!"}g}xNtd td D]8fd(d t| D}|t#j$|d)d*q|q.Wd1d t?|D}x:t|| D],\} |r| rj@+d2t-j.d#d3qWd4d t|| d D}!tjA|d |!d|d#d5}!|dk r*|B| x|CD]}"|!|"qWtDE}#|#jF}$|#jGfd6d7}%|#Hd#|#I|%||!}&|#H|$|#It|&}&WdQRXWdQRX|j o|dk o|jd k}'tJ|&}(|(rtK|&|'\})ntL|&|'\})tMjNrtO}*n d d8lPmQ}*|*jRSr>tTUr"t Vd9n|*R}+|+Wt!"|)| })|X|)Wd|Y|Z|[},X|,rt-.}-|-j j\]d:d|,D|+d;|-t!6|gF|rt#^}.|.j@}/t-j.t/0|d.}-|/+t_|-n t)j*dd t| D}0xt?|)D]\}"|"dkrdx t| D]}1|0|1dqHWq,t#j`|"| d?d*}2t!6>x6t| D]*}1|0|1tajb|2|1d@|1fd*qWWdQRXq,WfdAd |0D}0|.|0gS)Bay Builds graph operators that runs compilation and replicated computation. This is a lower level interface than replicate that returns a separate compile and execute output tensor. In the generated graph the compile op feeds into the execute op and no additional compilation is incurred when running the compile op before the execute op. The compile op returns additional information about the compilation but does not return the compiled program. Args: computation: A Python function that builds the computation to replicate. inputs: A list of lists of input tensors or `None` (equivalent to `[[]]`), indexed by `[replica_num][input_num]`. All replicas must have the same number of inputs. Each input can be a nested structure containing values that are convertible to tensors. Note that passing an N-dimension list of compatible values will result in a N-dimension list of scalar tensors rather than a single Rank-N tensors. If you need different behavior, convert part of inputs to tensors with `tf.convert_to_tensor`. infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to computation. device_assignment: If not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. Uses a default device assignment if `None`. The `DeviceAssignment` may be omitted if each replica of the computation uses only one core, and there is either only one replica, or the number of replicas is equal to the number of cores in the TPU system. name: (Deprecated) Does nothing. use_tpu: When false, the input `computation` is executed on the XLA CPU/GPU backends. Currently, only supports a default placement (computation is placed on GPU if one is available, and on CPU if not). maximum_shapes: A nested structure of tf.TensorShape representing the shape to which the respective component of each input element in each replica should be padded. Any unknown dimensions (e.g. tf.compat.v1.Dimension(None) in a tf.TensorShape or -1 in a tensor-like object) will be padded to the maximum size of that dimension over all replicas. The structure of `maximum_shapes` needs to be the same as `inputs[0]`. padding_spec: An enum specified by `tf.tpu.PaddingSpec`. This describes the padding policy when the `inputs` to `tf.tpu.replicate` is dynamic. One usage is to enable automatic bucketizing on the inputs by setting the value to `tpu.PaddingSpec.POWER_OF_TWO`, which can help to reduce the recompilation in the XLA side. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: A list of lists with the first list corresponding to the compile op and the second a list of output tensors, indexed by `[replica_num][output_num]`. Raises: ValueError: If all replicas do not have equal numbers of input tensors. ValueError: If the number of inputs per replica does not match the number of formal parameters to `computation`. ValueError: If the static `inputs` dimensions don't match with the values given in `maximum_shapes`. ValueError: If the structure of inputs per replica does not match the structure of `maximum_shapes`. N)rCr#rZallow_soft_placementzfAutomatic outside compilation is enabled. Ops without XLA kernels will be automatically placed on CPU.z@tpu.replicate() inputs must be a list of lists/tuples, received css|]}t|ttf VqdS)N)rKlisttuple)rinpr4r4r5rsz.split_compile_and_replicate..zGtpu.replicate() inputs must be a list of lists/tuples, received types: cSsg|] }t|qSr4)rb)rr%r4r4r5rsz/split_compile_and_replicate..rr;cSsg|]}tj|ddqS)T)r)r*r )rZper_replica_inputr4r4r5r)scSs t|ddS)NFT)r!)rr4r4r5rV/rWz-split_compile_and_replicate..cSs(g|] }|dkrtdnt|qS)Nr)rrrconvert_to_tensor)rrr4r4r5r6scSsg|] }|jqSr4)r)rrr4r4r5r;sz`Replicas must have the same number of inputs. Replica 0 had {} inputs, replica {} had {} inputs.cSsg|] }|jqSr4)r)rrr4r4r5rDszdReplicas must have matching input types. Replica 0 had input types {}, replica {} had input types {}zOSupplied computation cannot be called with the specified inputs. You specified z inputs: cSsg|] }|jqSr4)r=)rrr4r4r5rOsz, but the computation needs cSsg|] }|jqSr4)r=)rrr4r4r5rTs zand z: additional inputs from infeed, but the computation needs Fz9Dynamic input shapes are not supported with infeed queues)Z check_typescSsg|]\}}t||qSr4)r!)rryr4r4r5rlscSs"g|]}|dk rt|ndqS)N)r TensorShape)rrpr4r4r5rpsTz&TPU has inputs with dynamic shapes: %sZstep_marker_locationZSTEP_MARK_AT_ENTRYrcsg|]}|qSr4r4)rr) flat_inputsrr4r5rszinput{})r=Zcluster_rz/pivot)rp)r=rnro)rnr"cSs$g|]\}}tj|d|dqS)zreplicated_input_{})r=)rrAr)rrrr4r4r5rsZ_tpu_input_identity)rcSs g|]\}}|dkrdn|qS)Nr4)rZ replicatedr%r4r4r5rs)Z structureZ flat_sequencercsX|dd}|dk r*d|d<td||dkrB||f||S||f||SdS)z)Variables on TPU have a few restrictions. partitionerNz~Partitioned variables are not supported on TPU. Got `partitioner` that is %s for variable %s. Setting `partitioner` to `None`.)rr r)getterr=rrr+)saved_custom_getterr4r5 custom_getters z2split_compile_and_replicate..custom_getter) tensor_tracerz= 2.0 detected. Tensor Tracer v1 is not enabled.css|]}t|VqdS)N)r)r})rrr4r4r5rshost_compute_coreZcompilation_statuscsg|]}tjd|dqS)zshard_%d)r=)rgroup)rr) control_depsr4r5r.scSsg|]}gqSr4r4)rrr4r4r5r4szoutput{}zoutput_%d_shard_%dcsg|]}tj|ddqS)T)r)r*pack_sequence_as)rZ replica_outs) pack_templater4r5rMs)crrCZ serializedZcore_assignmentr tolistrrZget_soft_device_placementr inforKr# TypeErrorrbrYrrr*Zassert_same_structurer-Zconvert_variables_to_tensorsZ map_structurercrOrr Zcheck_function_argument_countZnumber_of_tuple_elementsziprrNrrrIr'rrr unique_namer=rno_opr_PIVOT_FOR_CLUSTERr r~r)r}rLrZtpu_replicate_metadatar%Ztpu_shard_contextr?rrr tf2xlaZset_dynamic_dimension_sizerr set_shaperr_r3Zset_number_of_shardsZgenerate_dequeue_opr!get_variable_scopeZ use_resourcer.Zset_use_resourceset_custom_getterZis_flat_postprocess_flat_outputs_postprocess_non_flat_outputstyping TYPE_CHECKINGrtensorflow.python.tpur/Z TensorTracer is_enabledrenabledwarnZ trace_tpuZ ExitResultrrrrprdZtpu_compilation_result_TPU_COMPILATION_STATUS_ATTRZtpu_replicated_outputrrA)3rrerr#r=r"rrrZmetadata_kwargsrnZflat_inputs_with_nonesZ is_compositer%Zflat_input_typesZ input_arityZflat_input_aritytypesZ arg_errorZdynamic_shape_inputsZflat_maximum_shapesZunpadded_inputsrrPZflat_replicated_inputsZreplicasZ cluster_namerormetadatarrZ compositeZcomputation_inputsrivscopeZsaved_use_resourcer.rneed_spmd_partitioningZoutputs_is_flatoutput_tensorsr/ttr0Z attr_valueZcompile_statusr_Zreplicated_outputsrZysr4)r2r*rr4r-r5rsnC            (                               . r)rrLr3c s|dkrt}tj|dd}tj|dd}|tf7}ddyH|rZfdd|D}n,ttd fd d|D}WdQRXWn0tk r}zt d |Wdd}~XYnXd d|D}d d|D}|||krt dt |dkr |ddt |}g}x|D]}|dkr0| dn|rbt |}|jdtjdd| |nRt|jrt|jntd 0t |}|jdtjdd| |WdQRXqW|||fS)aValidates non-flat outputs, add backs device assignments and other attrs. Args: outputs: Output from `computation` inside `tpu.rewrite`. need_spmd_partitioning: Whether XLA SPMD partitioning is needed. Returns: - Tensors extracted from outputs. - Operations extracted from outputs. - A pack template for use with nest.pack_sequence_as to pack the tensors. NF)rTcSs|dkr dSt|S)N)rr&)rr4r4r5rV|rWz+_postprocess_flat_outputs..cs$g|]}t|tjr|n|qSr4)rKrr)ro) maybe_convertr4r5rsz-_postprocess_flat_outputs..rcs$g|]}t|tjr|n|qSr4)rKrr)rrO)rPr4r5rsz_TPU function return values must all either be Operations or convertible to Tensors. Got error: cSsg|]}t|tjr|qSr4)rKrr)rrOr4r4r5rscSsg|]}t|tjs|qSr4)rKrr)rrOr4r4r5rszYTPU functions must return zero-or more Tensor values followed by zero or more Operations.r;_tpu_output_identity)r)r$r*r rr:rrr( ExceptionrOrrcrrAr_rr r~) rrLr4eZoutput_operationsrMZnew_output_tensorsrirOr4)rPr5r@TsH          r@c Cs,tj|dd}xt|D]\}}|dkr6d||<qt|tjrTtd|jdyt|}Wn2t k r}ztd|dWdd}~XYnX|rt |}|j dtjddt |||<qt|jr|jntd 4t |}|j dtjddt |||<WdQRXqW|g|fS) aValidates non-flat outputs, add backs device assignments and other attrs. Args: outputs: Output from `computation` inside `tpu.rewrite`. need_spmd_partitioning: Whether XLA SPMD partitioning is needed. Returns: - Tensors extracted from outputs. - An empty Operations list because Operations are not allowed in non-flat outputs. - A pack template for use with nest.pack_sequence_as to pack the tensors. T)rNztpu.rewrite does not support Operation as return value in non-flat output structure. You can set returned Operations as control dependencies of returned Tensors so Operations are triggered when Tensors are evaluated. Operation found: ""z`TPU function return values must all either be Operations or convertible to Tensors. Got error: "rQ)rr)r*r rrKrrrOr=r&rRrrAr_rr r~rr()rrLZ flat_outputsrrOrSr4r4r5rAs,    rAr;) rre num_shardsinput_shard_axesoutputs_from_all_shardsoutput_shard_axesrr#r=rr3c sdkrtd|dkr"gn|}t|tsBtdt|dd|D}|dkrfdgt|}t|t|krtdt|dt|d |rćfd dt||D} d dt| D} n gg} t|| |||| d \} } t| dtj r| | dgfSt| d}|dkr&dg|}|t|krNtd |dt|dt|t rd|g|}|t|krtd|dt|dg}xt||t| D]l\}}}|r|dj }|dk o|j dk}| |rtt|ntjt||dn| |dqW| |fS)a Shards `computation` for parallel execution. `inputs` must be a list of Tensors or None (equivalent to an empty list), each of which has a corresponding split axis (from `input_shard_axes`). Each input is split into `num_shards` pieces along the corresponding axis, and computation is applied to each shard in parallel. Tensors are broadcast to all shards if they are lexically captured by `computation`. e.g., x = tf.constant(7) def computation(): return x + 3 ... = shard(computation, ...) If `outputs_from_all_shards` is true, the outputs from all shards of `computation` are concatenated back together along their `output_shard_axes`. Otherwise, each output is taken from an arbitrary shard. Inputs and outputs of the computation must be at least rank-1 Tensors. Args: computation: A Python function that builds a computation to apply to each shard of the input. inputs: A list of input tensors or None (equivalent to an empty list). Each input tensor has a corresponding shard axes, given by `input_shard_axes`, which must have size divisible by `num_shards`. num_shards: The number of shards. input_shard_axes: A list of dimensions along which to shard `inputs`, or `None`. `None` means "shard all inputs along dimension 0". If not `None`, there must be one dimension per input. outputs_from_all_shards: Boolean or list of boolean. For each output, if `True`, outputs from all shards are concatenated along the corresponding `output_shard_axes` entry. Otherwise, each output is taken from an arbitrary shard. If the argument is a boolean, the argument's value is used for each output. output_shard_axes: A list of dimensions along which to concatenate the outputs of `computation`, or `None`. `None` means "concatenate all outputs along dimension 0". If not `None`, there must be one dimension per output. Ignored if `outputs_from_all_shards` is False. infeed_queue: If not `None`, the `InfeedQueue` to use to augment the inputs of `computation`. device_assignment: If not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. Uses a default device assignment if `None`. The `DeviceAssignment` may be omitted if each shard of the computation uses only one core, and there is either only one shard, or the number of shards is equal to the number of cores in the TPU system. name: (Deprecated) Does nothing. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: A tuple of (compile op, [output tensors]). Raises: ValueError: If num_shards <= 0 ValueError: If len(input_shard_axes) != len(inputs) ValueError: If len(output_shard_axes) != len(outputs from `computation`) rz0num_shards must be a positive integer. Received NzAtpu.shard()'s inputs must be a list of Tensors or None. Received cSsg|]}t|qSr4)rr&)rrr4r4r5rGsz+split_compile_and_shard..zKLength of input_shard_axes must be equal to the number of inputs. Received z inputs and z input_shard_axes.cs g|]\}}tj||dqS))r)rr)rrr)rUr4r5rTscSsg|] }t|qSr4)r#)rrr4r4r5rXs)rr#r=rzMLength of output_shard_axes must be equal to the number of outputs. Received z outputs and z output_shard_axes.zSLength of outputs_from_all_shards must be equal to the number of outputs. Received z outputs and z outputs_from_all_shards.)r)rOrKr#r7rbrr8rrrrlr Zndimsrcrrconcat)rrerUrVrWrXrr#r=rZ split_inputsZtransposed_inputsZ compile_oprZ num_outputsresultsrZ all_shardsrr Z is_scalarr4)rUr5split_compile_and_shardsZI           r[z tpu.shardc Cs t|||||||||| d dS)a Shards `computation` for parallel execution. `inputs` must be a list of Tensors or None (equivalent to an empty list), each of which has a corresponding split axis (from `input_shard_axes`). Each input is split into `num_shards` pieces along the corresponding axis, and computation is applied to each shard in parallel. Tensors are broadcast to all shards if they are lexically captured by `computation`. e.g., x = tf.constant(7) def computation(): return x + 3 ... = shard(computation, ...) TODO(phawkins): consider adding support for broadcasting Tensors passed as inputs. If `outputs_from_all_shards` is true, the outputs from all shards of `computation` are concatenated back together along their `output_shard_axes`. Otherwise, each output is taken from an arbitrary shard. Inputs and outputs of the computation must be at least rank-1 Tensors. Args: computation: A Python function that builds a computation to apply to each shard of the input. inputs: A list of input tensors or None (equivalent to an empty list). Each input tensor has a corresponding shard axes, given by `input_shard_axes`, which must have size divisible by `num_shards`. num_shards: The number of shards. input_shard_axes: A list of dimensions along which to shard `inputs`, or `None`. `None` means "shard all inputs along dimension 0". If not `None`, there must be one dimension per input. outputs_from_all_shards: Boolean or list of boolean. For each output, if `True`, outputs from all shards are concatenated along the corresponding `output_shard_axes` entry. Otherwise, each output is taken from an arbitrary shard. If the argument is a boolean, the argument's value is used for each output. output_shard_axes: A list of dimensions along which to concatenate the outputs of `computation`, or `None`. `None` means "concatenate all outputs along dimension 0". If not `None`, there must be one dimension per output. Ignored if `outputs_from_all_shards` is False. infeed_queue: If not `None`, the `InfeedQueue` to use to augment the inputs of `computation`. device_assignment: If not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. Uses a default device assignment if `None`. The `DeviceAssignment` may be omitted if each shard of the computation uses only one core, and there is either only one shard, or the number of shards is equal to the number of cores in the TPU system. name: (Deprecated) Does nothing. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: A list of output tensors. Raises: ValueError: If num_shards <= 0 ValueError: If len(input_shard_axes) != len(inputs) ValueError: If len(output_shard_axes) != len(outputs from `computation`) ) rerUrVrWrXrr#r=rr;)r[) rrerUrVrWrXrr#r=rr4r4r5shardsJr\ztpu.batch_parallel)rrerUrr#r=rc Cst|||||||dS)aShards `computation` along the batch dimension for parallel execution. Convenience wrapper around shard(). `inputs` must be a list of Tensors or None (equivalent to an empty list). Each input is split into `num_shards` pieces along the 0-th dimension, and computation is applied to each shard in parallel. Tensors are broadcast to all shards if they are lexically captured by `computation`. e.g., x = tf.constant(7) def computation(): return x + 3 ... = shard(computation, ...) The outputs from all shards are concatenated back together along their 0-th dimension. Inputs and outputs of the computation must be at least rank-1 Tensors. Args: computation: A Python function that builds a computation to apply to each shard of the input. inputs: A list of input tensors or None (equivalent to an empty list). The 0-th dimension of each Tensor must have size divisible by `num_shards`. num_shards: The number of shards. infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to `computation`. device_assignment: If not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. Uses a default device assignment if `None`. The `DeviceAssignment` may be omitted if each shard of the computation uses only one core, and there is either only one shard, or the number of shards is equal to the number of cores in the TPU system. name: (Deprecated) Does nothing. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: A list of output tensors. Raises: ValueError: If `num_shards <= 0` )rUrr#r=r)r\)rrerUrr#r=rr4r4r5batch_parallels5r]z tpu.rewrite)rrerr#r=rr3cCs&t||dkrdn|g||||ddS)aRewrites `computation` for execution on a TPU system. Args: computation: A Python function that builds a computation to apply to the input. If the function takes n inputs, 'inputs' should be a list of n tensors. `computation` may return a list of operations and tensors. Tensors must come before operations in the returned list. The return value of `rewrite` is a list of tensors corresponding to the tensors from the output of `computation`. All `Operation`s constructed during `computation` will be executed when evaluating any of the returned output tensors, not just the ones returned. inputs: A list of input tensors or `None` (equivalent to an empty list). Each input can be a nested structure containing values that are convertible to tensors. Note that passing an N-dimension list of compatible values will result in a N-dimension list of scalar tensors rather than a single Rank-N tensors. If you need different behavior, convert part of inputs to tensors with `tf.convert_to_tensor`. infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to `computation`. device_assignment: if not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. May be omitted for a single-core computation, in which case the core attached to task 0, TPU device 0 is used. name: (Deprecated) Does nothing. xla_options: An instance of `tpu.XLAOptions` which indicates the options passed to XLA compiler. Use `None` for default options. Returns: Same data structure as if computation(*inputs) is called directly with some exceptions for correctness. Exceptions include: 1) None output: a NoOp would be returned which control-depends on computation. 2) Single value output: A tuple containing the value would be returned. 3) Operation-only outputs: a NoOp would be returned which control-depends on computation. TODO(b/121383831): Investigate into removing these special cases. N)rr#r=rr)r)rrerr#r=rr4r4r5rewrite,s2r^ZReadVariableOpZAssignVariableOpZAssignAddVariableOpZAssignSubVariableOprZ VariableV2cCsht}xZ|rb|}x|r2t|tr*dS|j}qWt|tjrH|j}q t|t j r\|j }q dSq WdS)z6Check if it is currently under `_TPUInferenceContext`.TFN) rrIrJrK_TPUInferenceContextrMrZ _FuncGraphZ _outer_graphrrrH)rPrr4r4r5under_tpu_inference_contextus    r`csVeZdZdZdeedfdd ZddZdd Zd d Z d d Z e ddZ Z S)r_zA `ControlFlowContext` for nodes inside a TPU inference computation. The primary role of `_TPUInferenceContext` is to indicate the mode of operation and possibly sanity check operators inside a tpu.rewrite_for_inference() computation. T)r= check_opscstt|||_||_dS)N)rqr_rrr| _check_ops)rr=ra)rXr4r5rrsz_TPUInferenceContext.__init__cCs||dS)N)_AddOpInternal)rr_r4r4r5rsz_TPUInferenceContext.AddOpcCs@|jr*|jtkr*td|jd|jd|jr<|j|dS)NzOperation of type z (z) is not supported on the TPU for inference. Execution will fail if this op is used in the graph. Make sure your variables are using variable_scope.)rbrb_DENYLISTED_INFERENCE_OPSrr=rr)rr_r4r4r5rcs z#_TPUInferenceContext._AddOpInternalcCs|}|jr|j|}|S)N)rr)rrrr4r4r5rs z_TPUInferenceContext.AddValuecCs||dS)N)rc)rr_r4r4r5rsz_TPUInferenceContext.AddInnerOpcCsdS)Nr4)rr4r4r5rsz_TPUInferenceContext.grad_state)T)rSrrrrrlrrrrcrrrrrr4r4)rXr5r_s r_)rPcCs"tdd|DstddS)aValidates whether rewrite_for_inference() 'worked' for variables. The rewrite_for_inference() method is supposed to append GuaranteeConstOps after ReadVariableOps, but this mechanism works only if you are using tf.compat.v1.get_variable() to create and access variables in your tpu computation. This validation method can be called immediately after calling tpu.rewrite_for_inference() to check whether GuaranteeConstOps where added to the graph. Typical usages: tpu.validate_inference_rewrite_for_variables( tf.compat.v1.get_default_graph()) tpu.validate_inference_rewrite_for_variables(sess.graph) Args: graph: The graph which needs to be validated. Raises: RuntimeError: if validation failed. css|]}|jdkVqdS)ZGuaranteeConstN)rb)rrr4r4r5rsz;validate_inference_rewrite_for_variables..zNo GuaranteeConst ops found in the graph after running tpu.rewrite_for_inference(...). Please check that you are using tf.get_variable() to create and access variables in your tpu computation.N)rYget_operations RuntimeError)rPr4r4r5(validate_inference_rewrite_for_variablessrg)rrerr#r=r3cs(ddfdd}t|||||dS)aRewrites `computation` for inference on a TPU system. Other than 'rewriting' the computation to run on a TPU, if using variables in your computation, it moves the ReadVariableOps outside the TPU computation, and adds GuaranteeConst ops just after the ReadVariableOps. This mechanism works only if you are using tf.compat.v1.get_variable() to create and access variables in your tpu computation. You can validate whether this worked, by calling validate_inference_rewrite_for_variables() method immediately after this method to check whether GuaranteeConstOps where added to the graph. Args: computation: A Python function that builds a computation to apply to the input. If the function takes n inputs, 'inputs' should be a list of n tensors. If the function returns m outputs, rewrite will return a list of m tensors. inputs: A list of input tensors or `None` (equivalent to an empty list). infeed_queue: If not `None`, the `InfeedQueue` from which to append a tuple of arguments as inputs to `computation`. device_assignment: if not `None`, a `DeviceAssignment` describing the mapping between logical cores in the computation with physical cores in the TPU topology. May be omitted for a single-core computation, in which case the core attached to task 0, TPU device 0 is used. name: The name of the operator. Returns: A list of output tensors. c _s4td tj||f|||ddSQRXdS)Nz/GuaranteeConst)r=)rr?rZguarantee_const)r,r=rrr4r4r5guarantee_const_getters z5rewrite_for_inference..guarantee_const_gettercszttdd}zV|t}|j}|j}| | dd||}| || |Wd| X|S)z1Execute computation under `_TPUInferenceContext`.rewrite_for_inference)r=cSs|jS)N)r)r_r4r4r5rVrWzDrewrite_for_inference..wrapped_computation..N) r_rrIr9rr!r>r.Zcaching_devicer?Zset_caching_devicer)rrrrKZprev_custom_getterZprev_caching_devicer)rrhr4r5wrapped_computations    z2rewrite_for_inference..wrapped_computation)rerr#r=)r^)rrerr#r=rjr4)rrhr5ris"ri) prune_graphcCsx|gdd|jDD]r}t|tjs.qx^|D]R}|jtkrHq8d}x|jD]}| rTd}PqTW|s8t d|j |j| tq8WqWdS)aPrunes unconnected ops as listed in _UNCONNECTED_OPS_TO_PRUNE. Args: prune_graph: A tensorflow graph from which we wish to prune unconnected ops as listed in _UNCONNECTED_OPS_TO_PRUNE. In general, these ops should have no inputs and no consumers. These can often be left behind due to graph construction rewiring (for instance TF-Hub). While they never execute, they will cause XLA compile to fail so we strip them from XLA compile by removing the tpu_replicate attribute. cSsg|]}|qSr4r4)rfr4r4r5r sz2prune_unconnected_ops_from_xla..FTzGPruning OP %s of type %s from XLA Compile due to it being disconnected.N)Z _functionsvaluesrKrGraphrerb_UNCONNECTED_OPS_TO_PRUNErZ consumersr r6r=Z _clear_attrr)rkrPr_Zoutputs_consumedoutputr4r4r5prune_unconnected_ops_from_xla s"     rq)NNTN)N)N)NNNNNNN)N)NNNNTNNN) Nr;NTNNNNN) Nr;NTNNNNN)Nr;NNNN)NNNNN)NNNN)r collectionsenumrBrrrrrrrr Zabslr numpyr Z!tensorflow.compiler.tf2xla.pythonr r<Ztensorflow.core.frameworkr Ztensorflow.core.protobuf.tpur rrZ embedding_pb2Ztensorflow.pythonrZtensorflow.python.compiler.xlaZtensorflow.python.distributerrZtensorflow.python.frameworkrrrrrrrrrrrrrZtensorflow.python.opsrrr r!r"rDr#Zdevice_assignment_libr$r%r&Ztensorflow.python.tpu.opsr'Ztensorflow.python.typesr(rZtensorflow.python.utilr)r*r+r,r-Z tensorflow.python.util.tf_exportr.ZNotDifferentiablesetrrrorrrrHrr;r6ZTPUEmbeddingConfigurationrlrrDrrErGrQr\ZDeviceAssignmentr^Zregister_acd_resource_resolverZObjectIdentitySetrmZXLAControlFlowContextrLrrrIntEnumr namedtuplerZfilter_tracebackZ InfeedQueuerrr)rrr!rr@rArr[r\r]r^rdr`r_rnrgrirqr4r4r4r5sn(                                         6    :  y   Zc  $ -F,_,>` pK N6 N6(BA