B
    Y0dNJ                 @   s  d Z ddlZddlZddlZddlmZ ddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlmZmZmZ ddlZddlZddlZddlmZmZmZmZ ddlm  mZ ddlZddlm Z m!Z!m"Z" ddl#m$Z% ddl&m'Z'm(Z( e Z)ej*ej+d	gZ,ej-d
ddd Z.d|ddZ/dd Z0dd Z1dd Z2dd Z3dd Z4dd Z5dd Z6dd Z7ej89e:Z;ej8<e;d d!Z=eej8<e=d"d#Z>ej-e>d$d%d! Z?d&d' Z@d(d) ZAd*d+ ZBej+Cd,ejDeAd-d.ejDejEd/d.ejDejejEejFd0d1d.ejDejejEd2d0d3d.ejDejejEd4d0d5ej+jGe d6d7d8gej+d9d:d; ZHd<d= ZId>d? ZJd@dA ZKej-dBdC ZLG dDdE dEZMG dFdG dGZNej+CdHdIgdJe dKdLdMgfdNejOPdKdLdMgdOdPdQggfgdRdS ZQdTdU ZRej+CdVdWdXdYgdZd[ ZSeTd\d]d^ ZUG d_d` d`ejVZWdadb ZXdcdd ZYdedf ZZdgdh Z[ej+Cdidjdkdiedldmdmdngdodp Z\dqdr Z]dsdt Z^dudv Z_ej+Cdwej`ejFgdxdy Zadzd{ ZbdS )}a<  
manage legacy pickle tests

How to add pickle tests:

1. Install pandas version intended to output the pickle.

2. Execute "generate_legacy_storage_files.py" to create the pickle.
$ python generate_legacy_storage_files.py <output_dir> pickle

3. Move the created pickle to "data/legacy_pickle/<version>" directory.
    N)partial)Path)catch_warningsfilterwarningssimplefilter)PY38get_lzma_fileimport_lzmais_platform_little_endian)IndexSeriesperiod_range)DayMonthEndz1ignore:Timestamp.freq is deprecated:FutureWarningmodule)scopec           	   C   s2   ddl m}  t  tddtd |  S Q R X d S )Nr   )create_pickle_dataignorez The 'freq' argument in Timestamp)category)Z-pandas.tests.io.generate_legacy_storage_filesr   r   r   FutureWarning)r    r   M/var/www/html/venv/lib/python3.7/site-packages/pandas/tests/io/test_pickle.pycurrent_pickle_dataA   s
    r   c             C   s   t |trt||  d S |dr6tj}|| | n`|dkrv|tjkrX| tjksttq| |ksdt| j	|j	kstn t
td| dtj}|| | d S )NZsp_	timestampZassert_Z_equal)
isinstancer   tmassert_index_equal
startswithZassert_equalpdZNaTAssertionErrorfreqgetattrZassert_almost_equal)resultexpectedtypversion
comparatorr   r   r   compare_elementQ   s    


r'   c       
      C   sz   t |} t }xd|  D ]X\}}xN| D ]B\}}| | | }d| d| }	||	|d }	|	|||| q,W qW | S )NZcompare__r'   )r   read_pickleglobalsitemsget)
dataZvfr%   mr$   dvdtr"   r#   r&   r   r   r   compared   s    
r1   c             C   s   t | | | jj|jjks t| jjjr.tt | dk|dk | jj}|td tdksbt|tjdd }t	|tjst|tjdddkst|tjdd }t	|tjst|tjdddkstd S )Nr         )hours)daysr4   )nanoseconds)r5   r6   )
r   assert_series_equalindexr    r   	normalizer   r   Z	Timedeltar   )r"   r#   r$   r%   r    resr   r   r   compare_series_tsv   s    r;   c             C   s   t | | d S )N)r   r7   )r"   r#   r$   r%   r   r   r   compare_series_dt_tz   s    r<   c             C   s   t | | d S )N)r   r7   )r"   r#   r$   r%   r   r   r   compare_series_cat   s    r=   c             C   s   t | | d S )N)r   assert_frame_equal)r"   r#   r$   r%   r   r   r   compare_frame_dt_mixed_tzs   s    r?   c             C   s   t | | d S )N)r   r>   )r"   r#   r$   r%   r   r   r   compare_frame_cat_onecol   s    r@   c             C   s   t | ||| d S )N)r@   )r"   r#   r$   r%   r   r   r   compare_frame_cat_and_float   s    rA   c             C   sV   t | | t| jtst| jt ks,t| jdks:tt | d|d d S )NMr3   )r   r   r   r    r   r   Zfreqstrshift)r"   r#   r$   r%   r   r   r   compare_index_period   s
    rD   r-   legacy_pickle*z*.pickle)paramsc             C   s
   || j S )N)param)requestdatapathr   r   r   rE      s    c          	   C   sR   t  std tjtj|}tdd td t	| || W d Q R X d S )Nz"known failure on non-little endianT)recordr   )
r
   pytestskipospathbasenamedirnamer   r   r1   )r   rE   r%   r   r   r   test_pickles   s    
rR   c          	   C   s*   t |d}tj| |dd W d Q R X d S )Nwb)protocol)openpickledump)objrO   fhr   r   r   python_pickler   s    r[   c          	   C   s*   t | d}|d t|S Q R X d S )Nrbr   )rV   seekrW   load)rO   rZ   r   r   r   python_unpickler   s    
r_   pickle_writerpython)idZpandas_proto_default)rU   Zpandas_proto_highest   Zpandas_proto_4   Zpandas_proto_5zprotocol 5 not supported)reason)rb   Zmarksz5ignore:The 'freq' argument in Timestamp:FutureWarningc             C   s   | }x|  D ]\}}x|  D ]\}}xtjtgD ]}t }||| t|}	t|	|| t|}	t|	|| t	|dd}
||| |

d W d Q R X t	|dd}
t|
}	|

d W d Q R X t|	|| W d Q R X q4W q W qW d S )NrS   )moder   r\   )r+   r   	to_pickler[   r   ensure_cleanr)   r'   r_   rV   r]   )r   r`   r-   r$   r/   r0   r#   writerrO   r"   handler   r   r   test_round_trip_current   s"    




rk   c              C   s(   t  } t | jtj}t | | d S )N)r   makeDataFrameround_trip_pathlibrg   r   r)   r>   )dfr"   r   r   r   test_pickle_path_pathlib   s    ro   c              C   s(   t  } t | jtj}t | | d S )N)r   rl   Zround_trip_localpathrg   r   r)   r>   )rn   r"   r   r   r   test_pickle_path_localpath   s    rp   c          	   C   st   t t( tdt tj| dddddd W dQ R X t t( tdt tj| dddd	dd W dQ R X dS )
ap  

    Generated with

    >>> df = pd.DataFrame({"A": [1, 2, 3, 4], "B": [0, 0, 1, 1]}).to_sparse()
    >>> df.to_pickle("pandas/tests/io/data/pickle/sparseframe-0.20.3.pickle.gz",
    ...              compression="gzip")

    >>> s = df['B']
    >>> s.to_pickle("pandas/tests/io/data/pickle/sparseseries-0.20.3.pickle.gz",
    ...             compression="gzip")
    r   ior-   rW   zsparseseries-0.20.3.pickle.gzgzip)compressionNzsparseframe-0.20.3.pickle.gz)r   Zassert_produces_warningr   r   DeprecationWarningr   r)   )rJ   r   r   r   test_legacy_sparse_warning  s    

ru   c               C   s   dt d dS )N__
   z	__.pickle)r   Zrandsr   r   r   r   get_random_path"  s    rx   c            	   @   s   e Zd ZddddddZdd Zd	d
 Zejdddddgdd Z	ejddddddgdd Z
dd Zejdddddddgdd ZdS )TestCompressionz.nonez.gzz.bz2z.zipz.xz)Nrr   bz2zipxzc          
   C   s   |d krt || d S |dkr.t|d}n||dkrDt|d}nf|dkrtj|dtjd}|	|t
j| W d Q R X n*|dkrtt|d}nd| }t||dkrt|d$}| |	|  W d Q R X W d Q R X d S )	Nrr   wrz   r{   )rs   r|   zUnrecognized compression type: r\   )shutilcopyfilerr   rV   rz   BZ2FilezipfileZipFileZIP_DEFLATEDwriterN   rO   rP   r   lzma
ValueErrorread)selfZsrc_path	dest_pathrs   fmsgrZ   r   r   r   compress_file1  s"     
zTestCompression.compress_filec             C   s   |}|d }|d }t |}t |t}t  }|j||d t j||d*}	t|d}
|
|	  W d Q R X W d Q R X tj	|d d}t 
|| W d Q R X W d Q R X d S )Nz.compressedz.raw)rs   rS   )r   rh   rl   rg   decompress_filerV   r   r   r   r)   r>   )r   rs   rx   basepath1path2p1p2rn   r   rZ   df2r   r   r   test_write_explicitG  s    "z#TestCompression.test_write_explicitrs    NonebadZ7zc          
   C   sJ   t jtdd2 t|}t }|j||d W d Q R X W d Q R X d S )NzUnrecognized compression type)match)rs   )rL   Zraisesr   r   rh   rl   rg   )r   rs   rx   rO   rn   r   r   r   test_write_explicit_bad\  s    z'TestCompression.test_write_explicit_badextz.no_compressc             C   s   |}|| }|d }d }x"| j D ]}| j | |kr |}P q W t|}t|p}	t }
|
| tj||d*}t|	d}||  W d Q R X W d Q R X t	j
|	d d}t|
| W d Q R X W d Q R X d S )Nz.raw)rs   rS   )_compression_to_extensionr   rh   rl   rg   r   rV   r   r   r   r)   r>   )r   r   rx   r   r   r   rs   cr   r   rn   r   rZ   r   r   r   r   test_write_inferc  s     
"z TestCompression.test_write_inferc       
   
   C   s   |}|d }|d }t |\}t |F}t  }|j|d d | j|||d tj||d}	t ||	 W d Q R X W d Q R X d S )Nz.rawz.compressed)rs   )r   rh   rl   rg   r   r   r)   r>   )
r   rs   rx   r   r   r   r   r   rn   r   r   r   r   test_read_explicit~  s    z"TestCompression.test_read_explicitc          
   C   s   |}|d }|| }d }x"| j D ]}| j | |kr |}P q W t|X}t|B}	t }
|
j|d d | j||	|d t|	}t|
| W d Q R X W d Q R X d S )Nz.raw)rs   )	r   r   rh   rl   rg   r   r   r)   r>   )r   r   rx   r   r   r   rs   r   r   r   rn   r   r   r   r   test_read_infer  s    
zTestCompression.test_read_inferN)__name__
__module____qualname__r   r   r   rL   markparametrizer   r   r   r   r   r   r   r   ry   '  s    ry   c               @   s*   e Zd Zejdddddgdd ZdS )	TestProtocolrU   rT   r   r2   r3   c          	   C   sF   t |2}t  }|j||d t|}t || W d Q R X d S )N)rU   )r   rh   rl   rg   r   r)   r>   )r   rU   rx   rO   rn   r   r   r   r   	test_read  s
    
zTestProtocol.test_readN)r   r   r   rL   r   r   r   r   r   r   r   r     s   r   pickle_fileexcolsztest_py27.pklabr   ztest_mi_py27.pklABCc             C   s*   | ddd|}t |}t|j| d S )Nrq   r-   rW   )r   r)   r   r   columns)rJ   r   r   rO   rn   r   r   r   test_unicode_decode_error  s    
r   c           
   C   sl   t  Z} t  }t| d}|| W d Q R X t| d}t|}W d Q R X t || W d Q R X d S )NrS   r\   )r   rh   rl   rV   rg   r   r)   r>   )rO   rn   rZ   r"   r   r   r   test_pickle_buffer_roundtrip  s    
r   mockurlzhttp://url.comzftp://test.comzhttp://gzip.comc          	      sp   dd }G dd d t  H fdd}t  }|| | d| t|}t || W d Q R X d S )Nc          	   S   s*   t |d}tj| |dd W d Q R X d S )NrS   rT   )rU   )rV   rW   rX   )rY   rO   rZ   r   r   r   r[     s    z3test_pickle_generalurl_read.<locals>.python_picklerc               @   s4   e Zd Zdd Zdd Zdd Zdd Zd	d
 ZdS )z5test_pickle_generalurl_read.<locals>.MockReadResponsec             S   s.   t |d| _d|kr ddi| _n
dd i| _d S )Nr\   rr   zContent-Encoding)rV   fileheaders)r   rO   r   r   r   __init__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__init__c             S   s   | S )Nr   )r   r   r   r   	__enter__  s    z?test_pickle_generalurl_read.<locals>.MockReadResponse.__enter__c             W   s   |    d S )N)close)r   argsr   r   r   __exit__  s    z>test_pickle_generalurl_read.<locals>.MockReadResponse.__exit__c             S   s
   | j  S )N)r   r   )r   r   r   r   r     s    z:test_pickle_generalurl_read.<locals>.MockReadResponse.readc             S   s
   | j  S )N)r   r   )r   r   r   r   r     s    z;test_pickle_generalurl_read.<locals>.MockReadResponse.closeN)r   r   r   r   r   r   r   r   r   r   r   r   MockReadResponse  s
   r   c                 s    S )Nr   )r   kwargs)r   rO   r   r   mock_urlopen_read   s    z6test_pickle_generalurl_read.<locals>.mock_urlopen_readzurllib.request.urlopen)r   rh   rl   setattrr   r)   r>   )Zmonkeypatchr   r[   r   rn   r"   r   )r   rO   r   test_pickle_generalurl_read  s    


r   Zfsspecc           	   C   sD   t  2 d} t  }||  t| }t || W d Q R X d S )Nzmemory://afile)r   rh   rl   rg   r   r)   r>   )r   rn   r"   r   r   r   test_pickle_fsspec_roundtrip
  s    


r   c               @   s   e Zd Zdd ZdS )MyTzc             C   s   d S )Nr   )r   r   r   r   r     s    zMyTz.__init__N)r   r   r   r   r   r   r   r   r     s   r   c              C   sD   t tdt f} t| }t|d | d  t|d ts@td S )N)Zdtyper   r2   )r   objectr   r   round_trip_pickler7   r   r   )r#   r"   r   r   r   test_read_pickle_with_subclass  s    
r   c          	   C   s   t  }t   }|j|| d t| }W dQ R X t }|j|| d |d |	 |ksn| dksnt
tj|| d}|d t || dS )ze
    Read/write from binary file-objects w/wo compression.

    GH 26237, GH 29054, and GH 29570
    )rs   Nr   )rr   r{   )r   rl   rh   rg   r   
read_bytesrq   BytesIOr]   getvaluer   r   r)   r>   )rs   rn   rO   	referencebufferZread_dfr   r   r   %test_pickle_binary_object_compression"  s    


r   c             C   s8   | }|}dd }|| ||j  || ||j  d S )Nc             S   s   t | }t | | d S )N)r   r   r>   )frame	unpickledr   r   r   _test_roundtripD  s    
zDtest_pickle_dataframe_with_multilevel_index.<locals>._test_roundtrip)T)Z/multiindex_year_month_day_dataframe_random_dataZ multiindex_dataframe_random_dataZymdr   r   r   r   r   +test_pickle_dataframe_with_multilevel_index=  s    
r   c              C   sB   t dddd} ttjt| | }t|}|jj	dks>t
d S )Nz1/1/2011z1/1/2012rB   )r    )r   r   nprandomZrandnlenr   r   r8   r    r   )ZprngtsZnew_tsr   r   r   "test_pickle_timeseries_periodindexN  s    
r   namei	  g     H@i     )r2   r3   c             C   s$   t t j| d}|j| ks td S )N)r   )r   r   ZmakeTimeSeriesr   r   )r   r   r   r   r   test_pickle_preserve_nameV  s    r   c             C   s   t | }t ||  d S )N)r   r   r7   )Zdatetime_seriesZunp_tsr   r   r   test_pickle_datetimes_  s    
r   c             C   s   t | }t ||  d S )N)r   r   r7   )Zstring_seriesZ
unp_seriesr   r   r   test_pickle_stringsd  s    
r   c              C   sf   t tddjdg } t| }|jjd jdks:t	|jjd j
dksPt	t|dg |  d S )Nabcr   r   r2   )r2   T)r   listZastypeZilocr   r   Z_mgrblocksndimr   shaper7   )Zserr:   r   r   r    test_pickle_preserves_block_ndimi  s
    
r   rU   c             C   s@   t td}tt|j| |dtt j|d}t|| d S )Ni )rU   rs   )rs   )	r   	DataFrameranger   rm   r   rg   r)   r>   )rU   rs   rn   r"   r   r   r   %test_pickle_big_dataframe_compressionu  s
    r   c           	   C   sH   t jtdd} t| d}t|}W d Q R X t }t	
|| d S )Nz1.2.4zempty_frame_v1_2_4-GH#42345.pklr\   )rN   rO   joinlegacy_dirnamerV   rW   r^   r   r   r   r>   )rO   fdrn   r#   r   r   r   #test_pickle_frame_v124_unpickle_130  s
    r   )N)c__doc__rz   datetime	functoolsr   globrr   rq   rN   pathlibr   rW   r~   warningsr   r   r   r   numpyr   rL   Zpandas.compatr   r   r	   r
   Zpandas.util._test_decoratorsutilZ_test_decoratorstdZpandasr   r   r   r   Zpandas._testingZ_testingr   Zpandas.tseries.offsetsr   r   r   Z&skip_array_manager_not_yet_implementedr   Z
pytestmarkZfixturer   r'   r1   r;   r<   r=   r?   r@   rA   rD   rO   rQ   __file__herer   r   filesrE   rR   r[   r_   r   rH   rg   HIGHEST_PROTOCOLZskipifrk   ro   rp   ru   rx   ry   r   Z
MultiIndexZfrom_arraysr   r   r   Z
skip_if_nor   tzinfor   r   r   r   r   r   r   r   r   DEFAULT_PROTOCOLr   r   r   r   r   r   <module>   s   

! 

*'
	$