B
    ӻd                 @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlmZ ddlm Z  ddl!m"Z" ddl!m#Z# ddl!m$Z$ ddl%m&Z& ddl%m'Z' ddl%m(Z( ddl%m)Z) ddl*m+Z, ddl-m.Z. ddl/m0Z0 G dd de1ej2dZ3G dd  d e3Z4G d!d" d"e4Z5G d#d$ d$e3Z6G d%d& d&e3Z7G d'd( d(e3Z8G d)d* d*e3Z9G d+d, d,e3Z:G d-d. d.e:Z;e8e4e5e9e:e;e7e6gZ<d/d0 Z=d1d2 Z>d3d4 Z?d5d6 Z@d7d8 ZAG d9d: d:e1ZBG d;d< d<eBZCd=d> ZDd?d@ ZEdAdB ZFdCdD ZGe0dEg dFdGdH ZHe0dIg dFdXdJdKZIdYdLdMZJdNdO ZKdPdQ ZLdRdS ZMdTdU ZNdVdW ZOdS )ZzIAdapter module that convert different input data objects into tf.dataset.    N)cardinality)dataset_ops)iterator_ops)options)distribution_strategy_context)	input_lib)context)dtypes)errors)ops)
smart_cond)sparse_tensor)tensor_shape)backend)training_utils)
data_utils)dataset_creator)tf_utils)	array_ops)math_ops)
random_ops)
script_ops)
tf_logging)nest)keras_exportc               @   s   e Zd ZdZedddZejdddZejdd Z	ejd	d
 Z
ejdd Zdd Zejdd Zejdd Zejdd Zdd Zdd ZdS )DataAdaptera  Base class for input data adapter.

  In TF 2.0, tf.data is the preferred API for user to feed in data. In order
  to simplify the training code path, all the input data object will be
  converted to `tf.data.Dataset` if possible.

  Note that since this class is mainly targeted for TF 2.0, it might have a lot
  of assumptions under the hood, eg eager context by default, distribution
  strategy, etc. In the meantime, some legacy feature support might be dropped,
  eg, Iterator from dataset API in v1, etc.

  The sample usage of this class is like:

  ```
  x = tf.data.Dataset.range(100)
  adapter_cls = [NumpyArrayDataAdapter, ..., DatasetAdapter]
  applicable_adapters = [cls for cls in adapter_cls if cls.can_handle(x)]
  if len(applicable_adapters) != 1:
    raise ValueError("Expect only one adapter class to handle the input")

  dataset = applicable_adapters[0](x).get_dataset()
  for data in dataset:
    # training
  ```
  Nc             C   s   t dS )az  Whether the current DataAdapter could handle the input x and y.

    Structure wise, x and y can be single object, or list of objects if there
    multiple input/output, or dictionary of objects when the intput/output are
    named.

    Args:
      x: input features.
      y: target labels. Note that y could be None in the case of prediction.

    Returns:
      boolean
    N)NotImplementedError)xy r   ]/var/www/html/venv/lib/python3.7/site-packages/tensorflow/python/keras/engine/data_adapter.py
can_handleP   s    zDataAdapter.can_handlec             K   s$   |  ||s td| j||dS )a  Create a DataAdapter based on data inputs.

    The caller must make sure to call `can_handle()` first before invoking this
    method. Provide unsupported data type will result into unexpected behavior.

    Args:
      x: input features.
      y: target labels. Note that y could be None in the case of prediction.
      **kwargs: Other keyword arguments for DataAdapter during the construction
        of the tf.dataset.Dataset. For example:
        - Numpy data might have `sample_weights` which will be used for
          weighting the loss function during training.
        - Numpy data might need to have `batch_size` parameter when constructing
          the dataset and iterator.
        - Certain input might need to be distribution strategy aware. When
          `distribution_strategy` is passed, the created dataset need to respect
          the strategy.
        DataAdapter might choose to ignore any keyword argument if it doesn't
        use it, or raise exception if any required argument is not provide.
    z{} Cannot handle input {}, {}N)r!   
ValueErrorformat	__class__)selfr   r   kwargsr   r   r    __init__a   s    zDataAdapter.__init__c             C   s   t dS )a  Get a dataset instance for the current DataAdapter.

    Note that the dataset returned does not repeat for epoch, so caller might
    need to create new iterator for the same dataset at the beginning of the
    epoch. This behavior might change in future.

    Returns:
      An tf.dataset.Dataset. Caller might use the dataset in different
      context, eg iter(dataset) in eager to get the value directly, or in graph
      mode, provide the iterator tensor to Keras model function.
    N)r   )r%   r   r   r    get_dataset{   s    zDataAdapter.get_datasetc             C   s   t dS )a$  Return the size (number of batches) for the dataset created.

    For certain type of the data input, the number of batches is known, eg for
    Numpy data, the size is same as (number_of_element / batch_size). Whereas
    for dataset or python generator, the size is unknown since it may or may not
    have a end state.

    Returns:
      int, the number of batches for the dataset, or None if it is unknown. The
      caller could use this to control the loop of training, show progress bar,
      or handle unexpected StopIteration error.
    N)r   )r%   r   r   r    get_size   s    zDataAdapter.get_sizec             C   s   t dS )a5  Return the batch size of the dataset created.

    For certain type of the data input, the batch size is known, and even
    required, like numpy array. Where as for dataset, the batch is unknown
    unless we take a peek.

    Returns:
      int, the batch size of the dataset, or None if it is unknown.
    N)r   )r%   r   r   r    
batch_size   s    zDataAdapter.batch_sizec             C   s   |   S )aQ  Return a representative size for batches in the dataset.

    This is not guaranteed to be the batch size for all batches in the
    dataset. It just needs to be a rough approximation for batch sizes in
    the dataset.

    Returns:
      int, a representative size for batches found in the dataset,
      or None if it is unknown.
    )r*   )r%   r   r   r    representative_batch_size   s    z%DataAdapter.representative_batch_sizec             C   s   t dS )z1Whether the dataset has partial batch at the end.N)r   )r%   r   r   r    has_partial_batch   s    zDataAdapter.has_partial_batchc             C   s   t dS )zThe size of the final partial batch for dataset.

    Will return None if has_partial_batch is False or batch_size is None.
    N)r   )r%   r   r   r    partial_batch_size   s    zDataAdapter.partial_batch_sizec             C   s   t dS )z=Returns whether a new iterator should be created every epoch.N)r   )r%   r   r   r    should_recreate_iterator   s    z$DataAdapter.should_recreate_iteratorc             C   sD   |   r|  sdS |   |   }|  r@||  |   8 }|S )z1Returns number of samples in the data, or `None`.N)r)   r*   r,   r-   )r%   Ztotal_sampler   r   r    get_samples   s    zDataAdapter.get_samplesc             C   s   dS )zA hook called after each epoch.Nr   )r%   r   r   r    on_epoch_end   s    zDataAdapter.on_epoch_end)N)N)__name__
__module____qualname____doc__staticmethodr!   abcabstractmethodr'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r   r   r   r    r   5   s   	r   )	metaclassc                   sh   e Zd ZdZedddZd fdd	Zd	d
 Zdd Zdd Z	dd Z
dd Zdd Zdd Z  ZS )TensorLikeDataAdapterzEAdapter that handles Tensor-like objects, e.g. EagerTensor and NumPy.Nc                sH   t | }|d k	r |t |7 }t fdd t fdd|D S )Nc                s   t |  rdS dS )NTF)
isinstance)v)tensor_typesr   r    
_is_tensor   s    
z4TensorLikeDataAdapter.can_handle.<locals>._is_tensorc             3   s   | ]} |V  qd S )Nr   ).0r;   )r=   r   r    	<genexpr>   s    z3TensorLikeDataAdapter.can_handle.<locals>.<genexpr>)r   flatten_get_tensor_typesall)r   r   flat_inputsr   )r=   r<   r    r!      s    
z TensorLikeDataAdapter.can_handle   Fc	                sp  t tj||f|	 t|||f\}}}t||}tj|||dd\}}
}
t|||}tdd t	
|D  t|  s|rtt| nd tt  _ _t    _ttr܈ _tjd}dkr|}fdd	}||d} fd
d}||}||}dkrfdd }||}|_d S )NT)check_all_flatc             s   s   | ]}t |jd  V  qdS )r   N)intshape)r>   ir   r   r    r?      s    z1TensorLikeDataAdapter.__init__.<locals>.<genexpr>    rD   batchc                s*   t j tjd}r&dkr&t|}|S )N)dtyperJ   )r   ranger	   int64r   random_shuffle)_indices)num_samplesshuffler   r    permutation  s    
z3TensorLikeDataAdapter.__init__.<locals>.permutationc                s~     }t | dg|g}t | g}tj|}jrbtjt | |gjg}||}dkrz|	d
}|S )a  Convert a Tensor of indices into a dataset of batched indices.

      This step can be accomplished in several ways. The most natural is to
      slice the Tensor in a Dataset map. (With a condition on the upper index to
      handle the partial batch.) However it turns out that coercing the Tensor
      into a shape which is divisible by the batch size (and handling the last
      partial batch separately) allows for a much more favorable memory access
      pattern and improved performance.

      Args:
        indices: Tensor which determines the data order for an entire epoch.

      Returns:
        A Dataset of batched indices.
      r   rJ   i   )r   slicereshaper   	DatasetV2from_tensor_slices_partial_batch_sizefrom_tensorsconcatenaterR   repeat)rP   Znum_in_full_batchZfirst_k_indicesZflat_datasetZindex_remainder)r*   epochsnum_full_batchesr%   rR   r   r    slice_batch_indices-  s    

z;TensorLikeDataAdapter.__init__.<locals>.slice_batch_indicesc              W   s   t tj| S )N)r   map_structurer   rN   )rJ   r   r   r    shuffle_batchR  s    z5TensorLikeDataAdapter.__init__.<locals>.shuffle_batch) superr9   r'   _process_tensorlikebroadcast_sample_weight_modesr   handle_partial_sample_weightspack_x_y_sample_weightsetr   r@   pop_check_data_cardinalityrF   mathceil_size_batch_sizerX   r:   strlower_shuffler   rV   rL   r[   mapprefetchZflat_mapslice_inputs_dataset)r%   r   r   sample_weightssample_weight_modesr*   r\   stepsrR   r&   rO   inputsindices_datasetrS   r^   datasetr`   )r$   )r*   r\   r]   rQ   r%   rR   r    r'      s<    




 


zTensorLikeDataAdapter.__init__c             C   sb   t j|t j| f}dd }|j|t jd}t }d|j	_
| jrTtjj|_||}|S )a  Slice inputs into a Dataset of batches.

    Given a Dataset of batch indices and the unsliced inputs,
    this step slices the inputs in a parallelized fashion
    and produces a dataset of input batches.

    Args:
      indices_dataset: A Dataset of batched indices
      inputs: A python data structure that contains the inputs, targets,
        and possibly sample weights.

    Returns:
      A Dataset of input batches matching the batch indices.
    c                s   t  fdd|S )Nc                s   t j|  ddS )Nr   )axis)r   gather)d)rH   r   r    <lambda>m      zHTensorLikeDataAdapter.slice_inputs.<locals>.grab_batch.<locals>.<lambda>)r   r_   )rH   datar   )rH   r    
grab_batchl  s    z6TensorLikeDataAdapter.slice_inputs.<locals>.grab_batch)num_parallel_callsF)r   rV   ziprY   r[   rp   AUTOTUNEoptions_libOptionsZexperimental_optimizationZapply_default_optimizationsro   ZExternalStatePolicyZIGNOREZ"experimental_external_state_policyZwith_options)r%   rx   rw   ry   r   r   r   r   r    rr   X  s    

z"TensorLikeDataAdapter.slice_inputsc             C   s   | j S )N)rs   )r%   r   r   r    r(   }  s    z!TensorLikeDataAdapter.get_datasetc             C   s   | j S )N)rk   )r%   r   r   r    r)     s    zTensorLikeDataAdapter.get_sizec             C   s   | j S )N)rl   )r%   r   r   r    r*     s    z TensorLikeDataAdapter.batch_sizec             C   s
   | j dkS )Nr   )rX   )r%   r   r   r    r,     s    z'TensorLikeDataAdapter.has_partial_batchc             C   s
   | j pd S )N)rX   )r%   r   r   r    r-     s    z(TensorLikeDataAdapter.partial_batch_sizec             C   s   dS )NFr   )r%   r   r   r    r.     s    z.TensorLikeDataAdapter.should_recreate_iterator)N)NNNNrD   NF)r1   r2   r3   r4   r5   r!   r'   rr   r(   r)   r*   r,   r-   r.   __classcell__r   r   )r$   r    r9      s"         h%r9   c                   s6   e Zd ZdZed	ddZ fddZdd Z  ZS )
GenericArrayLikeDataAdaptera  Adapter that handles array-like data without forcing it into memory.

  This adapter handles array-like datasets that may be too big to fully
  fit into memory.

  Specifically, this adapter handles any Python class which implements:
  `__get_item__`, `__len__`, `shape`, and `dtype` with the same meanings
  as Numpy, but it ignores any case where all the inputs are Tensors or Numpy
  arrays (because that case is handled by the base TensorLikeDataAdapter).

  It ignores scipy sparse matrices and Composite Tensors because those are
  handled by the CompositeTensorDataAdapter.

  It also does not handle lists/tuples of scalars, because those are handled
  by the ListsOfScalarsDataAdapter.
  Nc                s^   t | }|d k	r |t |7 }dd  t| |sVt| |sVt fdd|D S dS d S )Nc             S   s(   t | do&t | do&t | do&t | dS )z6Return True if v is a Tensor, array, or is array-like.__getitem__rG   rK   __len__)hasattr)r;   r   r   r    _is_array_like  s    


z>GenericArrayLikeDataAdapter.can_handle.<locals>._is_array_likec             3   s   | ]} |V  qd S )Nr   )r>   r;   )r   r   r    r?     s    z9GenericArrayLikeDataAdapter.can_handle.<locals>.<genexpr>F)r   r@   r9   r!   CompositeTensorDataAdapterrB   )r   r   rC   r   )r   r    r!     s    
	z&GenericArrayLikeDataAdapter.can_handlec                s    t d tt| j|| d S )NzKeras is training/fitting/evaluating on array-like data. Keras may not be optimized for this format, so if your input data format is supported by TensorFlow I/O (https://github.com/tensorflow/io) we recommend using that to load a Dataset instead.)loggingwarningra   r   r'   )r%   argsr&   )r$   r   r    r'     s    z$GenericArrayLikeDataAdapter.__init__c                s`   t dd dd D d | jr8| jdkr8d  fdd	}|j|tjd
}|S )a  Slice inputs into a Dataset of batches.

    Given a Dataset of batch indices and the unsliced inputs,
    this step slices the inputs in a parallelized fashion
    and produces a dataset of input batches.

    Args:
      indices_dataset: A Dataset of batched indices
      inputs: A python data structure that contains the inputs, targets,
        and possibly sample weights.

    Returns:
      A Dataset of input batches matching the batch indices.
    c             S   s   t | j}d |d< t|S )Nr   )listrG   tuple)trG   r   r   r    dynamic_shape_like  s    
zDGenericArrayLikeDataAdapter.slice_inputs.<locals>.dynamic_shape_likec             S   s   g | ]
}|j qS r   )rK   )r>   inpr   r   r    
<listcomp>  s    z<GenericArrayLikeDataAdapter.slice_inputs.<locals>.<listcomp>TrJ   Fc                sP    fdd}t || g}x$t|D ]\}}|| q*W t|S )z%Grab a batch of data from the inputs.c                s     fddfddD S )Nc                s   t j|   dS )N)
contiguous)r   Zslice_arraysnumpy)r   )r   indr   r    slice_array  s    zdGenericArrayLikeDataAdapter.slice_inputs.<locals>.grab_batch.<locals>.py_method.<locals>.slice_arrayc                s   g | ]} |qS r   r   )r>   r   )r   r   r    r     s    zcGenericArrayLikeDataAdapter.slice_inputs.<locals>.grab_batch.<locals>.py_method.<locals>.<listcomp>r   )r   )r   rC   )r   r   r    	py_method  s    zOGenericArrayLikeDataAdapter.slice_inputs.<locals>.grab_batch.<locals>.py_method)r   Zeager_py_funcr   	set_shaper   pack_sequence_as)rP   r   Zflat_outr;   Zoriginal_inp)r   r   flat_dtypesrC   rw   r   r    r     s
    z<GenericArrayLikeDataAdapter.slice_inputs.<locals>.grab_batch)r   )r   r@   ro   rp   r   r   )r%   rx   rw   r   ry   r   )r   r   r   rC   rw   r    rr     s    
z(GenericArrayLikeDataAdapter.slice_inputs)N)	r1   r2   r3   r4   r5   r!   r'   rr   r   r   r   )r$   r    r     s
   	r   c                   s`   e Zd ZdZd fdd	ZedddZdd Zd	d
 Zdd Z	dd Z
dd Zdd Z  ZS )DatasetCreatorAdapterz'Adapter that handles dataset functions.Nc                sX   t t| j|f| t|tjs2tdt||d krBt	d|| _|| _
|| _d S )Nz\The input of a `DatasetCreatorAdapter` should be a `DatasetCreator` but it received type {}.zWhen using a `tf.keras.utils.experimental.DatasetCreator`, `steps_per_epoch`, `validation_steps` or `steps` argument must be provided in `Model.fit`, `Model.evaluate`, or `Model.predict`.)ra   r   r'   r:   r   DatasetCreator	TypeErrorr#   typer"   rv   strategy)r%   r   r   rv   distribution_strategyr&   )r$   r   r    r'     s    zDatasetCreatorAdapter.__init__c             C   s    t | tjr|d kstdS d S )NT)r:   r   r   AssertionError)r   r   r   r   r    r!     s    z DatasetCreatorAdapter.can_handlec             C   s   dS )NFr   )r%   r   r   r    r.     s    z.DatasetCreatorAdapter.should_recreate_iteratorc             C   s   d S )Nr   )r%   r   r   r    r)     s    zDatasetCreatorAdapter.get_sizec             C   s   | j j| j| jjdS )N)r   )r   !distribute_datasets_from_functionr   input_options)r%   r   r   r    r(     s    z!DatasetCreatorAdapter.get_datasetc             C   s
   t  d S )N)r   )r%   r   r   r    r*     s    z DatasetCreatorAdapter.batch_sizec             C   s
   t  d S )N)r   )r%   r   r   r    r,     s    z'DatasetCreatorAdapter.has_partial_batchc             C   s
   t  d S )N)r   )r%   r   r   r    r-     s    z(DatasetCreatorAdapter.partial_batch_size)NN)N)r1   r2   r3   r4   r'   r5   r!   r.   r)   r(   r*   r,   r-   r   r   r   )r$   r    r     s   r   c                   s`   e Zd ZdZedddZd fdd	Zdd	 Zd
d Zdd Z	dd Z
dd Zdd Z  ZS )r   z&Adapter that handles composite tensor.Nc                s`   t | }|d k	r |t |7 }dd   fddt fdd|D o^tfdd|D S )Nc             S   s0   t | r(t| tjtjfs(t| s(dS t| S )NT)	r   Zis_extension_typer:   r   rV   r   ZIteratorBase_is_distributed_dataset_is_scipy_sparse)r;   r   r   r    _is_composite+  s    
z<CompositeTensorDataAdapter.can_handle.<locals>._is_compositec                s   t | tjtjfrdS  | S )NT)r:   r   Tensornpndarray)r;   )r   r   r    _is_tensor_or_composite6  s    zFCompositeTensorDataAdapter.can_handle.<locals>._is_tensor_or_compositec             3   s   | ]} |V  qd S )Nr   )r>   r;   )r   r   r    r?   ;  s    z8CompositeTensorDataAdapter.can_handle.<locals>.<genexpr>c             3   s   | ]} |V  qd S )Nr   )r>   r;   )r   r   r    r?   <  s    )r   r@   anyrB   )r   r   rC   r   )r   r   r    r!   %  s    
z%CompositeTensorDataAdapter.can_handleFc                s  t t| j||f| t|||f\}}}t||}tj|||dd\}}	}	t|||}
tj	
|
}tt|d jd }|r||}|s|rtt|| nd}||}tt|| | _|| _| j|| k| _d | _| jr|| jd | j  | _|| _d S )NT)rE   r   rI   rD   )ra   r   r'   rb   rc   r   rd   re   r   rV   rW   rF   r   r@   rG   rR   ri   rj   rJ   rk   rl   _has_partial_batchrX   rs   )r%   r   r   rt   ru   r*   rv   rR   r&   rO   rw   ry   rQ   )r$   r   r    r'   >  s*    	

z#CompositeTensorDataAdapter.__init__c             C   s   | j S )N)rs   )r%   r   r   r    r(   h  s    z&CompositeTensorDataAdapter.get_datasetc             C   s   | j S )N)rk   )r%   r   r   r    r)   k  s    z#CompositeTensorDataAdapter.get_sizec             C   s   | j S )N)rl   )r%   r   r   r    r*   n  s    z%CompositeTensorDataAdapter.batch_sizec             C   s   | j S )N)r   )r%   r   r   r    r,   q  s    z,CompositeTensorDataAdapter.has_partial_batchc             C   s   | j S )N)rX   )r%   r   r   r    r-   t  s    z-CompositeTensorDataAdapter.partial_batch_sizec             C   s   dS )NTr   )r%   r   r   r    r.   w  s    z3CompositeTensorDataAdapter.should_recreate_iterator)N)NNNNNF)r1   r2   r3   r4   r5   r!   r'   r(   r)   r*   r,   r-   r.   r   r   r   )r$   r    r   "  s        #r   c                   sl   e Zd ZdZedddZedd Zd fdd		Zd
d Zdd Z	dd Z
dd Zdd Zdd Z  ZS )ListsOfScalarsDataAdapterzDAdapter that handles lists of scalars and lists of lists of scalars.Nc             C   s(   t | }d}|d k	r t |}|o&|S )NT)r   _is_list_of_scalars)r   r   Z	handles_xZ	handles_yr   r   r    r!   ~  s
    

z$ListsOfScalarsDataAdapter.can_handlec             C   s<   t | tttttfrdS t | ttfr8| r8t	| d S dS )NTr   F)
r:   floatrF   rm   bytes	bytearrayr   r   r   r   )r   r   r   r    r     s
    z-ListsOfScalarsDataAdapter._is_list_of_scalarsFc                sp   t t| j||f| t|}|d k	r2t|}|d k	rDt|}t||}t|f|||||d|| _d S )N)r   rt   ru   r*   rR   )ra   r   r'   r   Zasarrayrc   r9   _internal_adapter)r%   r   r   rt   ru   r*   rR   r&   )r$   r   r    r'     s     


z"ListsOfScalarsDataAdapter.__init__c             C   s
   | j  S )N)r   r(   )r%   r   r   r    r(     s    z%ListsOfScalarsDataAdapter.get_datasetc             C   s
   | j  S )N)r   r)   )r%   r   r   r    r)     s    z"ListsOfScalarsDataAdapter.get_sizec             C   s
   | j  S )N)r   r*   )r%   r   r   r    r*     s    z$ListsOfScalarsDataAdapter.batch_sizec             C   s
   | j  S )N)r   r,   )r%   r   r   r    r,     s    z+ListsOfScalarsDataAdapter.has_partial_batchc             C   s
   | j  S )N)r   r-   )r%   r   r   r    r-     s    z,ListsOfScalarsDataAdapter.partial_batch_sizec             C   s   dS )NTr   )r%   r   r   r    r.     s    z2ListsOfScalarsDataAdapter.should_recreate_iterator)N)NNNNF)r1   r2   r3   r4   r5   r!   r   r'   r(   r)   r*   r,   r-   r.   r   r   r   )r$   r    r   {  s   
    r   c                   sh   e Zd ZdZedddZd fdd	Zdd Zd	d
 Zdd Z	dd Z
dd Zdd Zdd Z  ZS )DatasetAdapterz'Adapter that handles `tf.data.Dataset`.Nc             C   s   t | tjtjfpt| S )N)r:   r   Z	DatasetV1rV   r   )r   r   r   r   r    r!     s    zDatasetAdapter.can_handlec                s4   t t| j||f| || _|| _| ||| d S )N)ra   r   r'   rs   _user_steps_validate_args)r%   r   r   rt   rv   r&   )r$   r   r    r'     s    zDatasetAdapter.__init__c             C   s   | j S )N)rs   )r%   r   r   r    r(     s    zDatasetAdapter.get_datasetc             C   s   d S )Nr   )r%   r   r   r    r)     s    zDatasetAdapter.get_sizec             C   s   d S )Nr   )r%   r   r   r    r*     s    zDatasetAdapter.batch_sizec             C   s   dS )NFr   )r%   r   r   r    r,     s    z DatasetAdapter.has_partial_batchc             C   s   d S )Nr   )r%   r   r   r    r-     s    z!DatasetAdapter.partial_batch_sizec             C   s.   t | jrdS | jd kp,t| j | jkS )NF)r   rs   r   r   r   )r%   r   r   r    r.     s    

z'DatasetAdapter.should_recreate_iteratorc             C   sh   t |stdt |s td|dkrdt| jr:tdt| j }|tjkrd|dkrdtddS )zValidates `__init__` arguments.z:`y` argument is not supported when using dataset as input.zF`sample_weight` argument is not supported when using dataset as input.NzRWhen providing a distributed dataset, you must specify the number of steps to run.zWhen providing an infinite dataset, you must specify the number of steps to run (if you did not intend to create an infinite dataset, make sure to not call `repeat()` on the dataset).)is_none_or_emptyr"   r   rs   r   r   INFINITE)r%   r   rt   rv   sizer   r   r    r     s    
zDatasetAdapter._validate_args)N)NNN)r1   r2   r3   r4   r5   r!   r'   r(   r)   r*   r,   r-   r.   r   r   r   r   )r$   r    r     s     r   c                   s   e Zd ZdZedddZd fdd		Zd
d Zedd Zdd Z	dd Z
dd Zdd Zdd Zdd Zdd Zdd Z  ZS ) GeneratorDataAdapterz5Adapter that handles python generators and iterators.Nc             C   s,   t | dst | do*t | do*t| tj S )N__next__next__iter__)r   r:   r   Sequence)r   r   r   r   r    r!   	  s    
zGeneratorDataAdapter.can_handlerD   F
   c                s*  | dd  t|stdt|s,tdttj||f| |\}	}|	}	t|	}	d k	rj	st
|	\}
}}jjfdd|
fd tt|	d jd _dd	 }t||	}td
d |	}||||  fdd}tjj|||d}|dkr |s |d}|_d S )NrR   zC`y` argument is not supported when using python generator as input.zO`sample_weight` argument is not supported when using python generator as input.c                s    | ddS )NF)Ztrainingr   )r   )modelr   r    r}   /  r~   z/GeneratorDataAdapter.__init__.<locals>.<lambda>)r   r   c             S   s,   | j }|jd kr|S tdd | D S )Nc             S   s   g | ]}d qS )Nr   )r>   rO   r   r   r    r   8  s    zMGeneratorDataAdapter.__init__.<locals>._get_dynamic_shape.<locals>.<listcomp>)rG   rankr   TensorShapeas_list)r   rG   r   r   r    _get_dynamic_shape3  s    
z9GeneratorDataAdapter.__init__.<locals>._get_dynamic_shapec             S   s   | j S )N)rK   )r   r   r   r    r}   ;  r~   c              3   s    x  D ]}  | V  qW d S )N)_standardize_batch)r   )generator_fnr%   r   r    wrapped_generatorB  s    z8GeneratorDataAdapter.__init__.<locals>.wrapped_generator)output_shapesrD   )rg   r   r"   ra   r   r'   _peek_and_restorer   rb   Zbuiltunpack_x_y_sample_weightZdistribute_strategyrunrF   r   r@   rG   _first_batch_sizer_   _handle_multiprocessingr   rV   Zfrom_generatorrq   rs   )r%   r   r   rt   workersuse_multiprocessingmax_queue_sizer   r&   peekZ
concrete_xrO   r   r   Zoutput_typesr   ry   )r$   )r   r   r%   r    r'     s2    


zGeneratorDataAdapter.__init__c             C   s<   t |\}}}t|||}t|}dd }t||}|S )z+Standardizes a batch output by a generator.c             S   s2   t | tjr.t| jjtjr.tj| t	 dS | S )N)rK   )
r:   r   r   
issubclassrK   r   floatingarrayr   floatx)r   r   r   r    _convert_dtypeV  s    z?GeneratorDataAdapter._standardize_batch.<locals>._convert_dtype)r   re   r   list_to_tupler_   )r%   r   r   r   sample_weightr   r   r   r    r   N  s    
z'GeneratorDataAdapter._standardize_batchc             C   s   t | }|t|g| fS )N)r   	itertoolschain)r   r   r   r   r    r   ^  s    z&GeneratorDataAdapter._peek_and_restorec                s8   dksdkr(r( fdd}nfdd}|S )z2Create a callable, possibly including an Enqueuer.rD   r   c                 s$   t jd} | j d |  S )N)r   )r   r   )r   ZGeneratorEnqueuerstartget)Zenqueuer)r   r   r   r   r   r    r   g  s    
zBGeneratorDataAdapter._handle_multiprocessing.<locals>.generator_fnc                  s    S )Nr   r   )r   r   r    r}   m  r~   z>GeneratorDataAdapter._handle_multiprocessing.<locals>.<lambda>r   )r%   r   r   r   r   r   r   )r   r   r   r   r    r   c  s    z,GeneratorDataAdapter._handle_multiprocessingc             C   s   | j S )N)rs   )r%   r   r   r    r(   p  s    z GeneratorDataAdapter.get_datasetc             C   s   d S )Nr   )r%   r   r   r    r)   s  s    zGeneratorDataAdapter.get_sizec             C   s   d S )Nr   )r%   r   r   r    r*   v  s    zGeneratorDataAdapter.batch_sizec             C   s   | j S )N)r   )r%   r   r   r    r+   y  s    z.GeneratorDataAdapter.representative_batch_sizec             C   s   dS )NFr   )r%   r   r   r    r,   |  s    z&GeneratorDataAdapter.has_partial_batchc             C   s   d S )Nr   )r%   r   r   r    r-     s    z'GeneratorDataAdapter.partial_batch_sizec             C   s   dS )NFr   )r%   r   r   r    r.     s    z-GeneratorDataAdapter.should_recreate_iterator)N)NNrD   Fr   N)r1   r2   r3   r4   r5   r!   r'   r   r   r   r(   r)   r*   r+   r,   r-   r.   r   r   r   )r$   r    r     s&        8r   c                   s\   e Zd ZdZedddZd fdd		Zed
d Zdd Zdd Z	dd Z
dd Z  ZS )KerasSequenceAdapterz,Adapter that handles `keras.utils.Sequence`.Nc             C   s   t | tjS )N)r:   r   r   )r   r   r   r   r    r!     s    zKerasSequenceAdapter.can_handleFrD   r   c	       
         sd   t |stdt |s tdt|| _|| _|| _d | _tt| j	|fd||||d|	 d S )NzI`y` argument is not supported when using `keras.utils.Sequence` as input.zU`sample_weight` argument is not supported when using `keras.utils.Sequence` as input.F)rR   r   r   r   r   )
r   r"   lenrk   _shuffle_sequence_keras_sequence	_enqueuerra   r   r'   )
r%   r   r   rt   rR   r   r   r   r   r&   )r$   r   r    r'     s     


zKerasSequenceAdapter.__init__c             C   s   | d | fS )Nr   r   )r   r   r   r    r     s    z&KerasSequenceAdapter._peek_and_restorec                s<   dksdkr*r* fdd}nfdd}|S )NrD   r   c                  s.   t jjd_jj d j S )N)r   rR   )r   r   )r   ZOrderedEnqueuerr   r   r   r   r   )r   r%   r   r   r   r   r    r     s
    zBKerasSequenceAdapter._handle_multiprocessing.<locals>.generator_fnc              3   s@   t t}  jr$t| } t|  x| D ]}| V  q*W d S )N)rL   r   r   r   randomrR   )orderrH   )r%   r   r   r    r     s    

r   )r%   r   r   r   r   r   r   )r   r%   r   r   r   r    r     s    
z,KerasSequenceAdapter._handle_multiprocessingc             C   s   | j S )N)rk   )r%   r   r   r    r)     s    zKerasSequenceAdapter.get_sizec             C   s   dS )NTr   )r%   r   r   r    r.     s    z-KerasSequenceAdapter.should_recreate_iteratorc             C   s   | j r| j   | j  d S )N)r   stopr   r0   )r%   r   r   r    r0     s    
z!KerasSequenceAdapter.on_epoch_end)N)NNFrD   Fr   N)r1   r2   r3   r4   r5   r!   r'   r   r   r)   r.   r0   r   r   r   )r$   r    r     s         r   c                s`    fddt D }|s2tdt tn&t|dkrXtd|t t|d S )z7Selects a data adapter than can handle a given x and y.c                s   g | ]}|  r|qS r   )r!   )r>   cls)r   r   r   r    r     s    z'select_data_adapter.<locals>.<listcomp>z9Failed to find data adapter that can handle input: {}, {}rD   zrData adapters should be mutually exclusive for handling inputs. Found multiple adapters {} to handle input: {}, {}r   )ALL_ADAPTER_CLSr"   r#   
_type_namer   RuntimeError)r   r   adapter_clsr   )r   r   r    select_data_adapter  s    r   c             C   s   t | trHtdd |  D }tdd |  D }dt| ||S t | ttfrxtdd | D }dt| |S t	t| S )z1Generates a description of the type of an object.c             s   s   | ]}t |V  qd S )N)r   )r>   keyr   r   r    r?     s    z_type_name.<locals>.<genexpr>c             s   s   | ]}t |V  qd S )N)r   )r>   r   r   r   r    r?     s    z%({} containing {} keys and {} values)c             s   s   | ]}t |V  qd S )N)r   )r>   valr   r   r    r?     s    z"({} containing values of types {}))
r:   dictrf   keysvaluesr#   r   r   r   rm   )r   Z	key_typesZ	val_typestypesr   r   r    r     s    
r   c             C   s   dd }t || } t | S )aX  Process tensor-like inputs.

  This function:

  (1) Converts `Numpy` arrays to `Tensor`s.
  (2) Converts `Scipy` sparse matrices to `SparseTensor`s.
  (2) Converts `list`s to `tuple`s (for `tf.data` support).

  Args:
    inputs: Structure of `Tensor`s, `NumPy` arrays, or tensor-like.

  Returns:
    Structure of `Tensor`s or tensor-like.
  c             S   sJ   t | tjr6d }t| jjtjr(t }t	j
| |dS t| rFt| S | S )N)rK   )r:   r   r   r   rK   r   r   r   r   r   "convert_to_tensor_v2_with_dispatchr   _scipy_sparse_to_sparse_tensor)r   rK   r   r   r    _convert_numpy_and_scipy  s    z5_process_tensorlike.<locals>._convert_numpy_and_scipy)r   r_   r   )rw   r   r   r   r    rb     s    
rb   c             C   s   | d kpt |  S )N)r   r@   )rw   r   r   r    r     s    r   c                s  | dkst | s S t trRt| tr@ fdd|  D S  fdd| D S  ryt t| t  W n t	t
fk
r
   tt dd | }tt dd  }y(t | t   td	|| W n* t	t
fk
r   t	d
||Y nX Y nX  S )z:Match sample_weight_modes structure with output structure.Nc                s   i | ]
} |qS r   r   )r>   r   )ru   r   r    
<dictcomp>&  s    z1broadcast_sample_weight_modes.<locals>.<dictcomp>c                s   g | ]} qS r   r   )r>   rO   )ru   r   r    r   '  s    z1broadcast_sample_weight_modes.<locals>.<listcomp>c             S   s   dS )Nz...r   )rO   r   r   r    r}   /  r~   z/broadcast_sample_weight_modes.<locals>.<lambda>c             S   s   dS )Nz...r   )rO   r   r   r    r}   0  r~   z8sample_weight modes were coerced from
  {}
    to  
  {}zVUnable to match target structure and sample_weight_modes structure:
  {}
    to  
  {})r   r@   r:   rm   r   r   Zassert_same_structurer   r   r"   r   r_   r   r   r   r#   )Ztarget_structureru   Z
target_strZmode_strr   )ru   r    rc     s2    

rc   c               @   s   e Zd ZdZd$dd	Zd
d Zdd Zejdd Z	dd Z
ejdd Zdd Zedd Zedd Zedd Zdd Zdd Zed d! Zd"d# ZdS )%DataHandlerz>Handles iterating over epoch-level `tf.data.Iterator` objects.Nr   rD   Fr   Tc             C   s   || _ || _d| _|| _|dkr.d| _d| _n|| _|  | _t||}||||||| |||
||t	
 |d| _t	
 }d| _| jd | _d| _| ||||	| dS )aO  Initializes a `DataHandler`.

    Arguments:
      x: See `Model.fit`.
      y: See `Model.fit`.
      sample_weight: See `Model.fit`.
      batch_size: See `Model.fit`.
      steps_per_epoch: See `Model.fit`.
      initial_epoch: See `Model.fit`.
      epochs: See `Model.fit`.
      shuffle: See `Model.fit`.
      class_weight: See `Model.fit`.
      max_queue_size: See `Model.fit`.
      workers: See `Model.fit`.
      use_multiprocessing: See `Model.fit`.
      model: The `Model` instance. Needed in order to correctly `build` the
        `Model` using generator-like inputs (see `GeneratorDataAdapter`).
      steps_per_execution: See `Model.compile`.
      distribute: Whether to distribute the `tf.dataset`.
        `PreprocessingLayer.adapt` does not support distributed datasets,
        `Model` should always set this to `True`.
    FNrD   )
r*   rv   r\   rt   rR   r   r   r   r   r   r   )_initial_epoch_epochs_insufficient_data_model_steps_per_execution_steps_per_execution_valuer   itemr   
ds_contextZget_strategy_adapter_current_step_step_increment%_configure_dataset_and_inferred_steps)r%   r   r   r   r*   steps_per_epochZinitial_epochr\   rR   class_weightr   r   r   r   Zsteps_per_execution
distributer   r   r   r   r    r'   F  s:    '


zDataHandler.__init__c             C   sT   ~| j  }|r|t|}| ||| _|rBt|sB||}|| _| 	  dS )z:Configure the `_dataset` and `_inferred_steps` attributes.N)
r   r(   rp   _make_class_weight_map_fn_infer_steps_inferred_stepsr   experimental_distribute_datasetrs   _validate_data_handler)r%   r   r   r  r  r  ry   r   r   r    r    s    

z1DataHandler._configure_dataset_and_inferred_stepsc          	   c   sj   |   X t| j}xFt| j| jD ]4}| jr0P | j rDt| j}||fV  | j	  q$W W dQ R X dS )z#Yields `(epoch, tf.data.Iterator)`.N)
_truncate_execution_to_epochiterrs   rL   r   r   r   r   r.   r0   )r%   Zdata_iteratorepochr   r   r    enumerate_epochs  s    




zDataHandler.enumerate_epochsc          	   c   s^   | j dk	o| j| j k}| j}z$|r8| j| j  | j | _dV  W d|rX| j| || _X dS )z3Truncates steps per execution to at most one epoch.N)r	  r   r   assign)r%   Zshould_truncateoriginal_valuer   r   r    r    s    

z(DataHandler._truncate_execution_to_epochc             C   s   t   d S )N)r   Z
async_wait)r%   r   r   r    sync  s    zDataHandler.syncc          	   c   sn   ydV  |    W nV ttjfk
rh   | jdkr<| j| _n(d| _| j| j }t	
d|| j  Y nX dS )z1Catches errors when an iterator runs out of data.NTzYour input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, {} batches). You may need to use the repeat() function when building your dataset.)r  StopIterationr
   ZOutOfRangeErrorr	  r  r   r   r   r   r   r#   )r%   Ztotal_epochsr   r   r    catch_stop_iteration  s    

z DataHandler.catch_stop_iterationc             c   s   d| _ x| jdks| j | jk r| jr&P | jdkpJ| jdkpJ| j| j  | jk}|rv| jd | _| j V  |  j | j7  _ q| j| j  }| j| |d | _| j V  |  j |7  _ | j| j qW dS )z#Yields steps for the current epoch.r   NrD   )r  r	  r   r   r  r   r  )r%   Zcan_run_full_executionZsteps_remainingr   r   r    rv     s&    



zDataHandler.stepsc             C   s   | j S )z<The number to increment the step for `on_batch_end` methods.)r  )r%   r   r   r    step_increment  s    zDataHandler.step_incrementc             C   s   | j S )ao  The inferred steps per epoch of the created `Dataset`.

    This will be `None` in the case where:

    (1) A `Dataset` of unknown cardinality was passed to the `DataHandler`, and
    (2) `steps_per_epoch` was not provided, and
    (3) The first epoch of iteration has not yet completed.

    Returns:
      The inferred steps per epoch of the created `Dataset`.
    )r	  )r%   r   r   r    inferred_steps  s    zDataHandler.inferred_stepsc             C   s
   | j d kS )N)r	  )r%   r   r   r    should_sync  s    zDataHandler.should_syncc             C   s   t d d S )NzThe training loop will run indefinitely since you have set `steps_per_epoch=-1`. Please use batch-level callbacks to save checkpoints or log training progress, etc)r   r   )r%   r   r   r     _log_indefinite_training_warning  s    z,DataHandler._log_indefinite_training_warningc             C   sr   |dkr|    dS |dk	r |S | j }|dk	r6|S t|}|tjkrZ|dkrZtd|dkrn|  S dS )z8Infers steps_per_epoch needed to loop through a dataset.Na!  When passing an infinitely repeating dataset, please specify a `steps_per_epoch` value so that epoch level callbacks continue to work. The value can be arbitrary, or a number that you think correctly defines the size of an epoch. Epoch-level callbacks will then be called at this interval.r   )r  r   r)   r   r   r"   r   r   )r%   rv   ry   Zadapter_stepsr   r   r   r    r    s    

zDataHandler._infer_stepsc             C   s
   | j  S )N)r   r/   )r%   r   r   r    _samples.  s    zDataHandler._samplesc             C   s    | j dkr| jd krtdd S )NrD   zrCould not infer the size of the data. With `steps_per_execution > 1`, you must specify the number of steps to run.)r   r	  r"   )r%   r   r   r    r  2  s    z"DataHandler._validate_data_handler)NNNNr   rD   FNr   rD   FNNT)r1   r2   r3   r4   r'   r  r  
contextlibcontextmanagerr  r  r  rv   propertyr  r  r  r  r  r  r  r   r   r   r    r   C  s8                
?r   c                   s:   e Zd ZdZd fdd	Zdd Zdd Zd	d
 Z  ZS )_ClusterCoordinatorDataHandlerz=A `DataHandler` that is compatible with `ClusterCoordinator`.Nc                s6   t |tjs| j||f|}t jf d|i| d S )Nr   )r:   r   r   _convert_to_dataset_creatorra   r'   )r%   r   r   r&   )r$   r   r    r'   >  s    z'_ClusterCoordinatorDataHandler.__init__c                s>    fdd}t t r2t t r2t|S tddS )z;Converts non-tf.data.Dataset to `DatasetCreator` instances.c                s$   ~ t }|f d  S )N)r   r   )r   r(   )Zinput_contextZdata_adapter_cls)r&   r   r   r   r    _dataset_fnG  s    
zO_ClusterCoordinatorDataHandler._convert_to_dataset_creator.<locals>._dataset_fnzOnly `tf.keras.utils.experimental.DatasetCreator`, `tf.Tensor`, numpy arrays and pandas dataframes are supported types at this time.N)r:   rA   r   r   r   )r%   r   r   r&   r   r   )r&   r   r   r    r  D  s    
z:_ClusterCoordinatorDataHandler._convert_to_dataset_creatorc                sT   t tjstd fdd}| jj|| _|dkrJd | _| 	  n|| _d S )NzEWhen using `ParameterServerStrategy`, `x` must be a `DatasetCreator`.c                  s    j jdS )N)r   )r   r   r   )r   r   r   r    per_worker_dataset_fn]  s    zc_ClusterCoordinatorDataHandler._configure_dataset_and_inferred_steps.<locals>.per_worker_dataset_fnr  )
r:   r   r   r   r   _cluster_coordinatorZcreate_per_worker_datasetrs   r	  r  )r%   r   r   r  r  r  r!  r   )r   r   r    r  W  s    
zD_ClusterCoordinatorDataHandler._configure_dataset_and_inferred_stepsc             C   s   | j j  d S )N)r   r"  join)r%   r   r   r    r  k  s    z#_ClusterCoordinatorDataHandler.sync)N)	r1   r2   r3   r4   r'   r  r  r  r   r   r   )r$   r    r  ;  s
   r  c              O   s$   t |d dd rt| |S t| |S )Nr   r"  )getattrr  r   )r   r&   r   r   r    get_data_handlero  s    
r%  c                sb   t t  }t tt|}||kr:d }t|t fdd|D fdd}|S )a  Applies class weighting to a `Dataset`.

  The `Dataset` is assumed to be in format `(x, y)` or `(x, y, sw)`, where
  `y` must be a single `Tensor`.

  Args:
    class_weight: A map where the keys are integer class ids and values are
      the class weights, e.g. `{0: 0.2, 1: 0.6, 2: 0.3}`

  Returns:
    A function that can be used with `tf.data.Dataset.map` to apply class
    weighting.
  zfExpected `class_weight` to be a dict with keys from 0 to one less than the number of classes, found {}c                s   g | ]} t | qS r   )rF   )r>   c)r  r   r    r     s    z-_make_class_weight_map_fn.<locals>.<listcomp>c                 s   t | \} }t r td jjdkr4tdt jjdkoTt d dk fdd fdd}t	|}|dk	rt
||j}t||f\}}|| }n|}| |fS )	z*Convert `class_weight` to `sample_weight`.zA`class_weight` is only supported for Models with a single output.   z8`class_weight` not supported for 3+ dimensional targets.rD   c                  s   t j ddS )NrD   )rz   )r   Zargmaxr   )r   r   r    r}     r~   zJ_make_class_weight_map_fn.<locals>._class_weights_map_fn.<locals>.<lambda>c                  s   t t dtjS )N)r  )r   castr   rU   r	   rM   r   )r   r   r    r}     r~   N)r   r   	is_nestedr"   rG   r   r   r   r   Z	gather_v2r   r(  rK   	expand_1d)r   r   swZ	y_classesZcw)class_weight_tensor)r   r    _class_weights_map_fn  s"    


z8_make_class_weight_map_fn.<locals>._class_weights_map_fn)	r   sortedr   rL   r   r#   r"   r   r   )r  Z	class_idsZexpected_class_ids	error_msgr-  r   )r  r,  r    r  u  s    r  c             C   s   dd }t || S )z=Expands 1-dimensional `Tensor`s into 2-dimensional `Tensor`s.c             S   s8   t | tjr4t | jtjr4| jjdkr4tj| ddS | S )NrD   r  )rz   )	r:   r   r   rG   r   r   r   r   Zexpand_dims_v2)r   r   r   r    _expand_single_1d_tensor  s    z+expand_1d.<locals>._expand_single_1d_tensor)r   r_   )r   r0  r   r   r    r*    s    r*  c                s   dd  t | } fdd|D }|r6td|tdd |D rP| | fS d}x|D ]}|dk	rZ|}P qZW t|jd	 }tt|d
|  }|d	ks||krtdj||ddd }t 	t
j|d	|d| }	t 	t
j|||d| }
|	|
fS )a  Split arrays into train and validation subsets in deterministic order.

  The last part of data will become validation data.

  Args:
    arrays: Tensors to split. Allowed inputs are arbitrarily nested structures
      of Tensors and NumPy arrays.
    validation_split: Float between 0 and 1. The proportion of the dataset to
      include in the validation split. The rest of the dataset will be included
      in the training split.
  Returns:
    `(train_arrays, validation_arrays)`
  c             S   s   t  }t| |p| d kS )N)rA   r:   )r   r<   r   r   r    
_can_split  s    z*train_validation_split.<locals>._can_splitc                s   g | ]} |st |qS r   )r   )r>   r   )r1  r   r    r     s    z*train_validation_split.<locals>.<listcomp>zh`validation_split` is only supported for Tensors or NumPy arrays, found following types in the input: {}c             s   s   | ]}|d kV  qd S )Nr   )r>   r   r   r   r    r?     s    z)train_validation_split.<locals>.<genexpr>Nr   g      ?zTraining data contains {batch_dim} samples, which is not sufficient to split it into a validation and training set as specified by `validation_split={validation_split}`. Either provide more data, or a different value for the `validation_split` argument.)	batch_dimvalidation_splitc             S   s   | d kr| S | || S )Nr   )r   r   endr   r   r    _split  s    z&train_validation_split.<locals>._split)r   r4  )r   r@   r"   r#   rB   rF   rG   ri   floorr_   	functoolspartial)Zarraysr3  Zflat_arraysZunsplitableZfirst_non_noner   r2  Zsplit_atr5  Ztrain_arraysZ
val_arraysr   )r1  r    train_validation_split  s4    

r9  z$keras.utils.unpack_x_y_sample_weight)Zv1c             C   s   t | ts| ddfS t| dkr.| d ddfS t| dkrL| d | d dfS t| dkrn| d | d | d fS d| }t|dS )a,  Unpacks user-provided data tuple.

  This is a convenience utility to be used when overriding
  `Model.train_step`, `Model.test_step`, or `Model.predict_step`.
  This utility makes it easy to support data of the form `(x,)`,
  `(x, y)`, or `(x, y, sample_weight)`.

  Standalone usage:

  >>> features_batch = tf.ones((10, 5))
  >>> labels_batch = tf.zeros((10, 5))
  >>> data = (features_batch, labels_batch)
  >>> # `y` and `sample_weight` will default to `None` if not provided.
  >>> x, y, sample_weight = tf.keras.utils.unpack_x_y_sample_weight(data)
  >>> sample_weight is None
  True

  Example in overridden `Model.train_step`:

  ```python
  class MyModel(tf.keras.Model):

    def train_step(self, data):
      # If `sample_weight` is not provided, all samples will be weighted
      # equally.
      x, y, sample_weight = tf.keras.utils.unpack_x_y_sample_weight(data)

      with tf.GradientTape() as tape:
        y_pred = self(x, training=True)
        loss = self.compiled_loss(
          y, y_pred, sample_weight, regularization_losses=self.losses)
        trainable_variables = self.trainable_variables
        gradients = tape.gradient(loss, trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, trainable_variables))

      self.compiled_metrics.update_state(y, y_pred, sample_weight)
      return {m.name: m.result() for m in self.metrics}
  ```

  Args:
    data: A tuple of the form `(x,)`, `(x, y)`, or `(x, y, sample_weight)`.

  Returns:
    The unpacked tuple, with `None`s for `y` and `sample_weight` if they are not
    provided.
  NrD   r   r'     z]Data is expected to be in format `x`, `(x,)`, `(x, y)`, or `(x, y, sample_weight)`, found: {})r:   r   r   r#   r"   )r   r/  r   r   r    r     s    0

r   z"keras.utils.pack_x_y_sample_weightc             C   s<   |dkrt | s| S | fS n|dkr.| |fS | ||fS dS )ao  Packs user-provided data into a tuple.

  This is a convenience utility for packing data into the tuple formats
  that `Model.fit` uses.

  Standalone usage:

  >>> x = tf.ones((10, 1))
  >>> data = tf.keras.utils.pack_x_y_sample_weight(x)
  >>> isinstance(data, tf.Tensor)
  True
  >>> y = tf.ones((10, 1))
  >>> data = tf.keras.utils.pack_x_y_sample_weight(x, y)
  >>> isinstance(data, tuple)
  True
  >>> x, y = data

  Args:
    x: Features to pass to `Model`.
    y: Ground-truth targets to pass to `Model`.
    sample_weight: Sample weight for each element.

  Returns:
    Tuple in the format used in `Model.fit`.
  N)r   r)  )r   r   r   r   r   r    re   4  s    
re   c             C   sx   t |||f\}}}|dkr$|f}n|dkr6||f}n
|||f}t| tj|}|rf|t|}| |}t|S )zCreates a single-batch dataset.N)	rb   rh   r   rV   rY   rp   r  r
  r  )r   r   r   r   r  r   ry   r   r   r    single_batch_iterator^  s    


r;  c             C   s   t dd t| D }t|dkr|d}xBtdddg| D ].\}}|d|d	d
d t|D 7 }q:W |d7 }t|d S )Nc             s   s   | ]}t |jd  V  qdS )r   N)rF   rG   )r>   rH   r   r   r    r?   u  s    z*_check_data_cardinality.<locals>.<genexpr>rD   zData cardinality is ambiguous:
r   r   r   z  {} sizes: {}
z, c             s   s   | ]}t |jd  V  qdS )r   N)rm   rG   )r>   rH   r   r   r    r?   z  s    z8Make sure all arrays contain the same number of samples.)rf   r   r@   r   r   r#   r#  r"   )r   rQ   msglabelZsingle_datar   r   r    rh   t  s    $rh   c              C   s>   ydd l } tjtj| j| jfS  tk
r8   tjtjfS X d S )Nr   )Zpandasr   r   r   r   ZSeriesZ	DataFrameImportError)pdr   r   r    rA     s
    rA   c             C   s.   yddl m} || S  tk
r(   dS X d S )Nr   )issparseF)Zscipy.sparser@  r>  )r   r@  r   r   r    r     s
    r   c             C   sv   |   }|j|j }}|j|j }}t|jjtj	rB|
t }tjtj|ddtj|ddfdd}t|||S )z1Converts a SciPy sparse matrix to a SparseTensor.rD   )rz   )Ztocoorowcolr   rG   r   rK   r   r   r   Zastyper   r   rZ   Zexpand_dimsr   ZSparseTensor)r   Z
sparse_coorA  rB  r   rG   rP   r   r   r    r     s    "r   c             C   s   t | tjS )N)r:   r   ZDistributedDatasetInterface)Zdsr   r   r    r     s    r   )NN)NNN)Pr4   r6   r  r7  r   ri   r   r   r   Z'tensorflow.python.data.experimental.opsr   Ztensorflow.python.data.opsr   r   r   r   Ztensorflow.python.distributer   r   r   Ztensorflow.python.eagerr   Ztensorflow.python.frameworkr	   r
   r   r   r   r   Ztensorflow.python.kerasr   Ztensorflow.python.keras.enginer   Ztensorflow.python.keras.utilsr   r   r   Ztensorflow.python.opsr   r   r   r   Ztensorflow.python.platformr   r   Ztensorflow.python.utilr   Z tensorflow.python.util.tf_exportr   objectABCMetar   r9   r   r   r   r   r   r   r   r   r   r   rb   r   rc   r   r  r%  r  r*  r9  r   re   r;  rh   rA   r   r   r   r   r   r   r    <module>   s      >`1Y@K L
	$ y48<>
+  
		