B
    ӻd:$                 @   s   d Z ddlmZ ddlmZ ddlmZ ddlmZ ddlm	Z	 ddlm
Z
 ddlmZ dd	lmZ dd
lmZ ddlmZ edG dd dejZdS )zNadam optimizer implementation.    )ops)backend_config)learning_rate_schedule)optimizer_v2)	array_ops)control_flow_ops)math_ops)	state_ops)	variables)keras_exportzkeras.optimizers.Nadamc                   sb   e Zd ZdZdZd fdd	Zd	d
 Zdd Z fddZdddZ	dddZ
 fddZ  ZS )Nadama  Optimizer that implements the NAdam algorithm.
  Much like Adam is essentially RMSprop with momentum, Nadam is Adam with
  Nesterov momentum.

  Args:
    learning_rate: A Tensor or a floating point value.  The learning rate.
    beta_1: A float value or a constant float tensor. The exponential decay
      rate for the 1st moment estimates.
    beta_2: A float value or a constant float tensor. The exponential decay
      rate for the exponentially weighted infinity norm.
    epsilon: A small constant for numerical stability.
    name: Optional name for the operations created when applying gradients.
      Defaults to `"Nadam"`.
    **kwargs: Keyword arguments. Allowed to be one of
      `"clipnorm"` or `"clipvalue"`.
      `"clipnorm"` (float) clips gradients by norm; `"clipvalue"` (float) clips
      gradients by value.

  Usage Example:
    >>> opt = tf.keras.optimizers.Nadam(learning_rate=0.2)
    >>> var1 = tf.Variable(10.0)
    >>> loss = lambda: (var1 ** 2) / 2.0
    >>> step_count = opt.minimize(loss, [var1]).numpy()
    >>> "{:.1f}".format(var1.numpy())
    9.8

  Reference:
    - [Dozat, 2015](http://cs229.stanford.edu/proj2015/054_report.pdf).
  TMbP??+?Hz>c                s   | dd|d< |d|}t|tjr0tdtt| j|f| | 	d|d| | 	d| j
 | 	d| | 	d| |pt | _d | _d S )	NZschedule_decaygMbp?decaylrzdThe Nadam optimizer does not support tf.keras.optimizers.LearningRateSchedules as the learning rate.learning_ratebeta_1beta_2)popget
isinstancer   ZLearningRateSchedule
ValueErrorsuperr   __init__Z
_set_hyper_initial_decayr   epsilon_m_cache)selfr   r   r   r   namekwargs)	__class__ \/var/www/html/venv/lib/python3.7/site-packages/tensorflow/python/keras/optimizer_v2/nadam.pyr   @   s    zNadam.__init__c             C   sx   |d j j}| jd kr@| jdg |ddtjjd| _| j| j x|D ]}| 	|d qFW x|D ]}| 	|d q`W d S )Nr   Zmomentum_cacheZonesF)shapedtypeZinitializerZ	trainableZaggregationmv)
r&   
base_dtyper   Z
add_weighttf_variablesZVariableAggregationZONLY_FIRST_REPLICAZ_weightsappendZadd_slot)r   var_list	var_dtypevarr#   r#   r$   _create_slotsW   s    


zNadam._create_slotsc             C   s<  t | d|}t | d|}t | d|}t| jd |}t| jd |}td|}	|ddt|	| j|    }
|ddt|	| j|    }t| j||
 }|| j	j
krt tj| j	|| jd	}|| }t|| t| j||||
|d| d| d|
 d| d| dt|| d
|||f< d S )Nr   r   r         gQ?g      ?g      ?)use_locking)lr_tneg_lr_tr   beta_1_tbeta_2_tm_tm_t_1one_minus_beta_1_tone_minus_beta_2_tone_minus_m_tone_minus_m_schedule_newone_minus_m_schedule_nextv_t_prime_denominator)r   identityZ
_get_hyperr   castZ
iterationspowr   _m_cache_readr   r&   r	   assign_use_lockingdictr   Z"convert_to_tensor_v2_with_dispatchr   )r   
var_devicer-   apply_stater3   r5   r6   Z
local_stepZ	next_stepZ
decay_baser7   r8   Zm_schedule_newZm_schedule_nextr#   r#   r$   _prepare_localj   s:    zNadam._prepare_localc                s   t | j| _tt| |S )N)r   r?   r   rB   r   r   _prepare)r   r,   )r"   r#   r$   rI      s    zNadam._prepareNc             C   s  |j |jj }}|pi ||fp,| ||}| |d}| |d}||d  }	|d | |d |  }
tj||
| jd}
|
|d  }|d | |d	 t	
|  }tj||| jd}||d
  }|d |	 |d |  }||d | t	||d    }tj||| jdjS )Nr'   r(   r<   r5   r9   )r2   r=   r6   r:   r>   r;   r8   r3   r   )devicer&   r)   r   _fallback_apply_stateget_slotr	   rC   rD   r   Zsquaresqrtop)r   gradr.   rG   rF   r-   coefficientsr'   r(   g_primer7   	m_t_primev_t	v_t_primem_t_barZvar_tr#   r#   r$   _resource_apply_dense   s&    


zNadam._resource_apply_densec          	   C   sr  |j |jj }}|pi ||fp,| ||}| |d}| |d}	||d  }
||d  }tj|||d  | jd}t	
|g  | |||}t||}W d Q R X ||d  }|d |
 |d	 |  }|| |d
  }tj|	|	|d  | jd}t	
|g  | |	||}t||}W d Q R X ||d  }t||d  }| |||d | | }tj|||g S )Nr'   r(   r<   r9   r5   )r2   r=   r;   r8   r:   r6   r>   r   r4   )rJ   r&   r)   r   rK   rL   r	   rC   rD   r   Zcontrol_dependenciesZ_resource_scatter_addr   gatherr   rM   r   group)r   rO   r.   indicesrG   rF   r-   rP   r'   r(   rQ   Zm_scaled_g_valuesr7   Z	m_t_slicerR   rU   Zv_scaled_g_valuesrS   Z	v_t_slicerT   Zv_prime_sqrt_plus_epsZ
var_updater#   r#   r$   _resource_apply_sparse   s6    


zNadam._resource_apply_sparsec                s>   t t|  }|| d| j| d| d| jd |S )Nr   r   r   )r   r   r   r   r   )r   r   
get_configupdateZ_serialize_hyperparameterr   r   )r   config)r"   r#   r$   r[      s    zNadam.get_config)r   r   r   r   r   )N)N)__name__
__module____qualname____doc__Z_HAS_AGGREGATE_GRADr   r/   rH   rI   rV   rZ   r[   __classcell__r#   r#   )r"   r$   r      s       $

(r   N)ra   Ztensorflow.python.frameworkr   Ztensorflow.python.kerasr   Z$tensorflow.python.keras.optimizer_v2r   r   Ztensorflow.python.opsr   r   r   r	   r
   r*   Z tensorflow.python.util.tf_exportr   ZOptimizerV2r   r#   r#   r#   r$   <module>   s   