B
    Z0dh                 @   sL  d dl Z d dl mZ d dlZd dlmZ d dlZd dlZd dlm	Z	 d dl
m  mZ d dlZd dlmZmZmZmZmZmZmZ d dlmZmZmZ ejjZejddd	 Z d
d Z!dd Z"dd Z#dd Z$dd Z%dd Z&dd Z'dd Z(dd Z)dd Z*ej+dd Z,d d! Z-d"d# Z.d$d% Z/ejj0d&d'd(d)d* Z1d+d, Z2dS )-    N)	timedelta)catch_warnings)	Timestamp)	DataFrame
MultiIndexSeries_testingconcat
date_rangeread_hdf)_maybe_removeensure_clean_pathensure_clean_storez7ignore:object name:tables.exceptions.NaturalNameWarningc             C   sp  t | \}tddD t }t|d |d|d d  |d|dd   t|d | t|d |jd|d d dd |d|dd   t|d | t|d |d	|d d  |d	|dd   t|d | t|d
 |d
|d d  |d
|dd   t|d | tddgddgddgddggd}d|d< t	j
|jd< t|d |d| t|d | ttt	jjddddt	jdtt	jjddddt	jdtt	jjddddt	jdtdddddgt	jddt	dd }t|d! |d!| t|d! | t|d! |jd!|d"d#d$gd% t|d! | W d Q R X W d Q R X d S )&NT)recorddf1
   df2table)formatdf3z/df3z/df3 foozdf3 foo      r   )datatestingmixed_column)r   r   df      )highsize)dtypei  i   @l           l          @l            l            l            )u08u16u32Zu64)indexZuintsr!   r"   r#   )data_columns)r   r   tmmakeTimeDataFramer   appendassert_frame_equalputr   npnanlocr   randomrandintZuint8Zuint16Zuint32Zuint64arange)
setup_pathstorer   Z	uint_data r3   V/var/www/html/venv/lib/python3.7/site-packages/pandas/tests/io/pytables/test_append.pytest_append   sT    



"


r5   c       	   	   C   s  t | }t }t }ttd}|d| |d }t|| |j	d ksXt
|d| |d }t|| |j	d kst
d|_	|d| |d }t|| |j	|j	kst
||dk }|dd}t|| ||dk|jd	k @  }|dd
}t|| ttjdddgd}tt||d< d|d< d|jdddf< |jddgdd | }|jd|_|d| t|d | W d Q R X d S )Nd   sstsfoons<   zfoo>60F   Z   zfoo>70 and index<90r   r   A)columnsBCbar   T)inplacer   mi)r   r&   ZmakeStringSeriesZmakeTimeSeriesr   r+   r0   r(   assert_series_equalnameAssertionErrorselectr$   r   r.   randnlenr-   	set_indexstackZ	droplevel)	r1   r2   r7   r8   r:   resultexpectedrE   sr3   r3   r4   test_append_seriese   s@    rQ   c             C   s  t | }tttjddtjdtjdddtdtdddd	d	d
t	dd}t
|d tj|jd	dddddgf< |d|d d  |d|dd   t|d | | }tj|jd d df< t
|d |d|d d  |d|dd   t|d | | }tj|jd d df< t
|d |d|d d  |d|dd   t|d | | }tj|jd d df< t
|d |d|d d  |d|dd   t|d | W d Q R X d S )N   Zint32r9   rB   20010101i  r   r   r   )r>   A1A2r@   rA   DE)r$   r      rT   r@   rV   rW   r   rU   r   r   )r   r   r   r+   r.   rJ   astyper   datetimer0   r   r,   r-   r(   r&   r)   copy)r1   r2   r   r   r   r   r3   r3   r4   test_append_some_nans   sB    





r\   c             C   s`  t | L}ttjdtjddtdd}tj|jddd d f< t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d |d
d   t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d | tdd t|d |	d|d d  |	d|dd   t
|d | tdd t|d |	d|d d  |	d|dd   t
|d |d
d   ttjdtjddddtdd}tj|jddd d f< t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d | t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d | ttjdtjdddtdtddddddtdd}tj|jddd d f< t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d | t|d |j	d|d d dd	 |j	d|dd  dd	 t
|d | W d Q R X d S )NrR   )rT   rU   )r$   r   rX   r   r   T)dropnar   Fzio.hdf.dropna_tabler   Zdf4r9   rB   )rT   rU   r@   rA   rS   i  r   r   )rT   rU   r@   rA   rV   rW   )r   r   r+   r.   rJ   r0   r,   r-   r   r(   r&   r)   pdZ
set_optionr   rZ   )r1   r2   r   r3   r3   r4   test_append_all_nans   sn    











r`   c          
   C   s  t |  }t }|jd |_t|d |jd|jd d d df dgd |d|jd d dd f  t|d | |	dd}|j
dgd}t|| |	dd}|j
dg|jd	d
 d}t|| td}tjt|d |	dd W d Q R X W d Q R X d S )Nr   r   r?   )Zaxesz	columns=Ar>   )r?   )z	columns=Azindex=df.index[0:4]r      )r?   r$   zOpassing a filterable condition to a non-table indexer [Filter: Not Initialized])matchzcolumns=A and index>df.index[4])r   r&   r'   r$   Z
_with_freqr   r(   ilocr)   rI   reindexreescapepytestraises	TypeError)r1   r2   r   rN   rO   msgr3   r3   r4   !test_append_frame_column_oriented  s"    
$rk   c          
   C   s   t | }xtdD ]}ttjddtdd}td|d< |d  |d 7  < tdgt| dd|d< tdgt| d	d|d	< |d d
kr|d= tdgt| dd|d< |d d
kr|	d}||d< |j
ddd |d| qW W d Q R X t | }ttjddtddd}tdgt| dd|d< tdgt| d	d|d	< |d| tdgt| d	d|d< td}tjt|d |d| W d Q R X tdgt| dd|d< td}tjt|d |d| W d Q R X W d Q R X d S )Nr   r   ZAB)r?   r$   r   Zint64)r    Zint16r   rC   r>   T)rD   r   Zfloat64)r?   r    Zint16_2zCcannot match existing table structure for [int16] on appending data)rb   g      ?Zfloat_3zAcannot match existing table structure for [A,B] on appending data)r   ranger   r+   r.   rJ   listr   rK   poprL   r(   re   rf   rg   rh   
ValueError)r1   r2   ir   arj   r3   r3   r4   )test_append_with_different_block_ordering6  s<    


rr   c                s  t |  tdd  fdd}tddgddgg} d	| t d	| |d	d
d tddgddgg} d	| t||g}t d	| |d	d
d tddgddgg} jd|ddid t d| |dd
d  d| tddgddgg}d}tj	t
|d  d| W d Q R X t d} jd|d ddid t d|d   jd|d dddid t d|d   jd |d!dd"id# |  jd$d%d} d | t d t||g  jd&|d d!dd"id#  d&|d  t d&t|d |d g t d' t }d(|d)< tj|j|jd*d d)f< d+|d,< tj|j|jdd- d,f< d.|d/< tj|j|jd*d  d/f<  d'|  d'}t|| W d Q R X W d Q R X t |   fd0d}td(d+d1td2d3}t d'  jd'|d4d5id |d'd4d5  d'jd4gksjtt d'  jd'|dgd4d5id |d'd4d5  d'jdd4gkstt d'  jd'|dgdd5id |d'dd5 |d'd6d5  d'jdgkstt d'  jd'|d d7 d5d  jd'|d7d  d5d t d' | td(d(d(d8d8d8gd4gd9}t d' td:}tj	t
|d  jd'|d;d;d<d W d Q R X W d Q R X d S )=NT)r   c                s"   t  | jj|j|kstd S )N)getattr
get_storerr   descriptionitemsizerH   )keyrG   r   )r2   r3   r4   	check_colm  s    z+test_append_with_strings.<locals>.check_col{   Z	asdqwertyiY  ZdggnhebbsdfbdfbZdf_bigvalues_block_1rX   |   ZasdqyiZ  ZdggnhefbdfbZdf_big2values2   )min_itemsizedf_newZ
abcdefqhijZabcdefghijklmnopqrtsuvwxyzzTrying to store a string with len \[26\] in \[values_block_1\] column but\nthis column has a limit of \[15\]!\nConsider using min_itemsize to preset the sizes on these columns)rb   rA   r7   r@   r$   ra   Zss2)r%   r~   Zss3r      )r   r~   Zlonger)rA   Zss4r   r9   stringr   rB   string2   ZbahZstring3c                s"   t  | jj|jst|d S )N)rs   rt   r   ru   rv   rH   )rw   rG   r   )r2   r3   r4   rx     s    )r>   r@   r   )r$   r>      Zvalues_block_0r   Zbarh)r?   zBmin_itemsize has the key [foo] which is not an axis or data_columnrR   )r9   Zfoobar)r   r   r   r(   r&   r)   rI   r	   rg   rh   ro   ZmakeMixedDataFramerL   rF   r*   r[   Zreset_indexZassignr   r'   r+   r,   r-   r$   rl   rt   r%   rH   re   rf   )r1   rx   r   r   rO   r   rj   rN   r3   )r2   r4   test_append_with_stringsh  s    "

 




r   c          
   C   s|   t | j}tddddddddgi}|jd	|d d
 ddid |jd	|d
d  ddid t|d	| W d Q R X d S )Nxrq   bcdef r   r   )r~   )r   r   r(   r&   r)   rI   )r1   r2   r   r3   r3   r4   test_append_with_empty_string  s
    
r   c       	   	      sn  t |  t }d|jd|jdf< t d  jd|d d dgd  d|dd   t d |  j	j
jjjjjdkst j	j
jjjjjdkst dd}||jdk }t||  dd	}|j|jd
d  d}||jdk }t|| | }d|d< tj|j|jdd
 df< d|j|jdd df< t d  jd|dgd  dd}||jdk }t||  fdd}W d Q R X t |  t d  jd|dgddid |ddd t d  jd|dgdd |ddd t d  jd|dgddid |ddd W d Q R X t | j d|d< d|d< d|d< t d  jd|ddgdddd d |ddd |ddd |dd!d W d Q R X t | H | }d|jd|jd"f< d#|jd|jdf< d|d< |jd}tj|jdd
|f< d|jdd|f< d|d< |jd}tj|jdd|f< d|jd$d%|f< t d  jd|d"dddgd  dd&}||jdk|jdk@ |jdk@ |jdk @  }tj||d'd(  dd)}||jdk|jd*k@  }t|| W d Q R X t |  | }d|d< tj|j|jd
d df< d|j|jd$d+ df< d*|d< td,|d-< |jdd.}tj|j|jd/d d"dd-gf< t d0  jd0|dd1ddd-gd  d0d}||jdk }t||  d0d2d3d4g}||jdk|jdk@ |jdk@  }tj||d'd( W d Q R X t | ( tjd5 t d6d%d7}t!tj"d%d/|d"dd1gd8}d|d< tj|j|jd
d df< d|j|jd$d+ df< |jd d dd1gf # |jd d dd1gf< d*|d<  jd0|dd1ddgd  d0d}||jdk }t||  d0d2d3d9g}||jdk|jdk@ |jdk@  }t|| W d Q R X d S ):Ng      ?r   r@   r   r   )r%   TzB>0zB>0 and index>df.index[3]ra   )r$   r9   r   r   rB   r   r   zstring='foo'c                s"   t  | jj|j|kstd S )N)rs   rt   r   ru   rv   rH   )rw   rG   r   )r2   r3   r4   rx     s    z0test_append_with_data_columns.<locals>.check_col   )r%   r~   r|   Z	foobarbahr   Z
foobarbah1Zstring_block1Z
foobarbah2Zstring_block2(   r}   )r   r   r|   rz   r>   g         r   z.string='foo' and string2='foo' and A>0 and B<0F)Z
check_freqzstring='foo' and string2='cool'cool	   Z20010102rZ   )rZ   rC   df_dcrA   zB > 0zC > 0zstring == fooi  z1/1/2000)Zperiods)r$   r?   zstring == "foo")$r   r&   r'   rc   r?   Zget_locr   r(   r)   Z_handlerootr   r   colsr$   Z
is_indexedrH   r@   rI   rd   r[   r+   r,   r-   r   r   r>   r   _convertrA   r.   seedr
   r   rJ   abs)	r1   r   rN   rO   r   rx   slr   r$   r3   )r2   r4   test_append_with_data_columns  s    








* 
","r   c             C   s(  t ddddgdddggdddd	d	d
d
dddg
dd	d
dd	d	d
dd	d
g
gddgd}ttjdd|dddgd}t| V}|d| |d}t	|| |jdddgd}|j
ddgd}t	|| W d Q R X tdD}|j|ddd t|dddgd}|j
ddgd}t	|| W d Q R X d S )Nr9   rB   ZbazZquxonetwothreer   r   r   rC   )Zlevelscodesnamesr   r>   r@   rA   )r$   r?   rE   )r?   ztest.hdfr   r   )r   )r   r   r+   r.   rJ   r   r(   rI   r&   r)   rd   r   Zto_hdfr   )r1   r$   r   r2   rN   rO   pathr3   r3   r4   test_append_hierarchical  s"    .


r   c          
      s  t  Z}t }|jd|dd |d}t|| |jd|dd |d}t|| W d Q R X  fdd}t }d	|d
< d|d< |d d|d< |d dk|d< td|d< td|d< ||tj t  }tt	dd}|d| t
jtdd |d W d Q R X ttjddt	dd}|d| t|d| |d| t|d| tt	dd}|d| t|d| W d Q R X d S )Nr   r   )	chunksizer   r   )Zexpectedrowsc          
      sN   xHdD ]@}t  dd*}|jd| |d |d}|||  W d Q R X qW d S )N)r   r   i  w)modeobj)r   )r   r(   rI   )r   Z
comparatorr   r2   rN   )r1   r3   r4   check  s
    

ztest_append_misc.<locals>.checkr9   r   g      ?Zfloat322Zfloat32r   bool20130101Ztime1Z20130102Ztime2ABC)r?   z 'No object named df in the file')rb   rC   r   )r   r&   makeDataFramer(   rI   r)   rY   r   r   rm   rg   rh   KeyErrorr+   r.   Zrandr*   )r1   r2   r   rN   r   Zdf_emptyr3   )r1   r4   test_append_misc  s<    



r   c          
   C   sX  t | D}t }dggt| |d< |jd tjks<tt	d}t
jt|d |d| W d Q R X dggt| |d< dggt| |d< t
jt|d |d| W d Q R X t }ttdd	d
|jd}|t}tj|dd< ||d< |jd tjkstd}t
jt|d |d| W d Q R X d}t
jt|d |dtd W d Q R X t	d}t
jt|d |dttd W d Q R X t }|d| d|d< t	d}t
jt|d |d| W d Q R X t|d td|d< |d| d|d< t	d}t
jt|d |d| W d Q R X W d Q R X d S )Nrq   invalidziCannot serialize the column [invalid]
because its data contents are not [string] but [mixed] object dtype)rb   r   Zinvalid2Zinvalid3i  r   r   )r$   r   r   z>too many timezones in this block, create separate data columnsz(value must be None, Series, or DataFramer   z]cannot properly create the storer for: [group->df,value-><class 'pandas.core.series.Series'>]r9   zinvalid combination of [non_index_axes] on appending data [(1, ['A', 'B', 'C', 'D', 'foo'])] vs current table [(1, ['A', 'B', 'C', 'D'])]r   rB   zinvalid combination of [values_axes] on appending data [name->values_block_1,cname->values_block_1,dtype->bytes24,kind->string,shape->(1, 30)] vs current table [name->values_block_1,cname->values_block_1,dtype->datetime64,kind->datetime64,shape->None])r   r&   r   rK   Zdtypesr+   Zobject_rH   re   rf   rg   rh   ri   r(   r   rZ   r$   rY   objectr,   r0   ro   r   r   )r1   r2   r   rj   rP   r3   r3   r4   test_append_raise  sV    
 
r   c          	   C   s~  t tddd tdD d}|d |d  |d< tj|jd	d
df< t| "}t|d |jd|dd |	d}t
|| |j	ddd}t
|| |j	ddd}t
||jd	d   |	dd}t
||jd	d   |	dd}|jdgd}t
||jdd   |	dd}|jdd  }t
||jdd   t|d |d| |	d}t
|| W d Q R X d S )Nr   c             S   s    g | ]}t d t|dd qS )r   r   )daysseconds)r   r   ).0rp   r3   r3   r4   
<listcomp>&  s    z.test_append_with_timedelta.<locals>.<listcomp>r   )r>   r@   r>   r@   rA   rC   r   r   T)r%   zC<100000)wherezC<pd.Timedelta('-3D')zC<'-3D'zC<'-500000s')Zsubsetr   z	C<'-3.5D'r   ra   r   )r   r   rl   r+   r,   r-   r   r   r(   rI   r&   r)   rc   r]   r*   )r1   r   r2   rN   r3   r3   r4   test_append_with_timedelta  s4    



r   c          
   C   s4  t  }t  jdjd}d|d< t||gdd}t| }d}tjt|d  |j	d	d
gd d|dd W d Q R X tjt|d |j	d d d|dd W d Q R X d}tjt|d |	d|d W d Q R X |j	d	d
gd d|dd |j
ddgddgdd}||jdk|jdk@  }t || W d Q R X d S )Nz{}_2)r?   rB   r9   r   )axisz=append_to_multiple requires a selector that is in passed dict)rb   r>   r@   )r   r   r   )selectorzQappend_to_multiple must have a dictionary specified as the way to split the valuer   r   zA>0zB>0)r   r   r   )r&   r'   renamer   r	   r   rg   rh   ro   append_to_multipleselect_as_multipler>   r@   r)   )r1   r   r   r   r2   rj   rN   rO   r3   r3   r4   test_append_to_multipleO  s&    
  r   c          	   C   s   t  }t  jdjd}tj|jd|jddgf< t	||gdd}t
| `}|jddgd d|dd	d
 |ddg}| }t || t |dj|dj W d Q R X d S )Nz{}_2)r?   r   r>   r@   )r   )r   r   r   T)r   r]   r   )r&   r'   r   r   r+   r,   rc   r?   get_indexerr	   r   r   r   r]   r)   Zassert_index_equalrI   r$   )r1   r   r   r   r2   rN   rO   r3   r3   r4   test_append_to_multiple_dropnaq  s    
r   Fz8append_to_multiple_dropna_false is not raising as failed)runreasonc          
   C   s   t  }t  jdjd}tj|jd|jddgf< t	||gdd}t
| l}|jddgd d|dd	d
 d}tjt|d |ddg W d Q R X |dj|djrtW d Q R X d S )Nz{}_2)r?   r   r>   r@   )r   )df1adf2ar   F)r   r]   z;Cannot select as multiple after appending with dropna=False)rb   r   )r&   r'   r   r   r+   r,   rc   r?   r   r	   r   r   rg   rh   ro   r   rI   r$   equalsrH   )r1   r   r   r   r2   rj   r3   r3   r4   $test_append_to_multiple_dropna_false  s    
r   c          	   C   s   t tddtddtddd dd tdD dd tdD d}|jd	g }t| V}|jd
gddgddgd|jd	g dddddd |dddg}t	|| W d Q R X d S )Nr      X   c             S   s   g | ]}d qS )rq   r3   )r   _r3   r3   r4   r     s    z8test_append_to_multiple_min_itemsize.<locals>.<listcomp>rR   c             S   s   g | ]}d qS )Zabcder3   )r   r   r3   r3   r4   r     s    )IXNumBigNumStrLongStrr   r   r   r   r   r   )r$   numsstrsr$   r   r6   r   )r   r   r   )r~   r   r   )
r   r+   r0   rl   rc   r   r   r   r&   r)   )r1   r   rO   r2   rN   r3   r3   r4   $test_append_to_multiple_min_itemsize  s"    




r   )3rZ   r   re   warningsr   numpyr+   rg   Zpandas._libs.tslibsr   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr_   r   r   r   r   r&   r	   r
   r   Zpandas.tests.io.pytables.commonr   r   r   markZsingleZ
pytestmarkfilterwarningsr5   rQ   r\   r`   rk   rr   r   r   r   r   r   Z&skip_array_manager_not_yet_implementedr   r   r   r   Zxfailr   r   r3   r3   r3   r4   <module>   s<   $	F//U2y 6N1"