B
    ӻd                 @   s|   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddl	m
Z dd Zd	d
 Zdd Zdd Zdd Zdd Zdd ZdS )zOptimizer utilities.    )central_storage_strategy)distribution_strategy_context)reduce_util)clip_ops)
tf_loggingc             C   s   t | } t| }|rVt r@dd |D }t jtjj	|}qZt
 jt|fd}ng }g }d}xB| D ]:\}}|dkr|d|f qh||| |f |d7 }qhW |t|kstd|S )zReturns all-reduced gradients aggregated via summation.

  Args:
    grads_and_vars: List of (gradient, variable) pairs.

  Returns:
    List of (gradient, variable) pairs where gradients have been all-reduced.
  c             S   s   g | ]}|d  qS )r    ).0pairr   r   \/var/www/html/venv/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/utils.py
<listcomp>%   s    z,all_reduce_sum_gradients.<locals>.<listcomp>)argsr   N   zFailed to add all gradients)listfilter_empty_gradientsstrategy_supports_no_merge_calldistribute_ctxget_strategyextendedZ_replica_ctx_all_reduceds_reduce_utilReduceOpSUMZget_replica_contextZ
merge_call_all_reduce_sum_fnappendlenAssertionError)grads_and_varsZfiltered_grads_and_varsgradsZreducedZreduced_with_nonesZreduced_posgvr   r   r
   all_reduce_sum_gradients   s&    	
r   c             C   s   t | } | s| S g }g }x2| D ]*\}}|dkr:|| q|||f qW t |}|sptddd | D f |rtddd |D  |S )zDFilter out `(grad, var)` pairs that have a gradient equal to `None`.Nz+No gradients provided for any variable: %s.c             S   s   g | ]\}}|j qS r   )name)r   _r   r   r   r
   r   L   s    z*filter_empty_gradients.<locals>.<listcomp>zAGradients do not exist for variables %s when minimizing the loss.c             S   s   g | ]
}|j qS r   )r    )r   r   r   r   r
   r   P   s    )tupler   
ValueErrorloggingwarning)r   filteredZvars_with_empty_gradsZgradvarr   r   r
   r   ;   s$    r   c                s     dkrdd S  fdd}|S )z@Creates a gradient transformation function for clipping by norm.Nc             S   s   | S )Nr   )r   r   r   r
   <lambda>W       z+make_gradient_clipnorm_fn.<locals>.<lambda>c                s4   t t tjtjfrtd fdd| D }|S )Nz:`clipnorm` is not supported with `CenteralStorageStrategy`c                s    g | ]\}}t | |fqS r   )r   Zclip_by_norm)r   r   r   )clipnormr   r
   r   b   s    zKmake_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fn.<locals>.<listcomp>)
isinstancer   r   r   CentralStorageStrategyCentralStorageStrategyV1r#   )r   clipped_grads_and_vars)r*   r   r
   gradient_clipnorm_fnY   s    
z7make_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   )r*   r/   r   )r*   r
   make_gradient_clipnorm_fnT   s    r0   c                s     dkrdd S  fdd}|S )z@Creates a gradient transformation function for clipping by norm.Nc             S   s   | S )Nr   )r   r   r   r
   r(   l   r)   z2make_global_gradient_clipnorm_fn.<locals>.<lambda>c                sL   t t tjtjfrtdt|  \}}t	| \}}t
t||}|S )NzA`global_clipnorm` is not supported with `CenteralStorageStrategy`)r+   r   r   r   r,   r-   r#   zipr   Zclip_by_global_normr   )r   r   	variablesZclipped_gradsr!   r.   )r*   r   r
   r/   n   s    
z>make_global_gradient_clipnorm_fn.<locals>.gradient_clipnorm_fnr   )r*   r/   r   )r*   r
    make_global_gradient_clipnorm_fni   s    r3   c                s     dkrdd S  fdd}|S )zACreates a gradient transformation function for clipping by value.Nc             S   s   | S )Nr   )r   r   r   r
   r(      r)   z,make_gradient_clipvalue_fn.<locals>.<lambda>c                s4   t t tjtjfrtd fdd| D }|S )Nz;`clipvalue` is not supported with `CenteralStorageStrategy`c                s$   g | ]\}}t |   |fqS r   )r   Zclip_by_value)r   r   r   )	clipvaluer   r
   r      s   zMmake_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fn.<locals>.<listcomp>)r+   r   r   r   r,   r-   r#   )r   r.   )r4   r   r
   gradient_clipvalue_fn   s    

z9make_gradient_clipvalue_fn.<locals>.gradient_clipvalue_fnr   )r4   r5   r   )r4   r
   make_gradient_clipvalue_fn~   s    r6   c             C   s   | j tjj|S )N)r   Zbatch_reduce_tor   r   r   )distributionr   r   r   r
   r      s    r   c              C   s    t  sdS t  } | j  S )zDReturns if the current Strategy can operate in pure replica context.T)r   Zhas_strategyr   r   Z_use_merge_call)Zstrategyr   r   r
   r      s    r   N)__doc__Ztensorflow.python.distributer   r   r   r   r   Ztensorflow.python.opsr   Ztensorflow.python.platformr   r$   r   r   r0   r3   r6   r   r   r   r   r   r
   <module>   s   #