B
    0dX              	   @   s  d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZmZ ddlmZmZ ddlmZ ddlmZmZ ddlmZmZ dd	lmZ dd
lmZmZ ddlmZmZ ddlm Z m!Z! ddlm"Z" ddl#m$Z$m%Z%m&Z& ddl'm(Z( dddZ)dd Z*dd Z+dd Z,dd Z-e$dddgdd Z.d d! Z/d"d# Z0d$d% Z1d&d' Z2d(d) Z3d*d+ Z4d,d- Z5d.d/ Z6g fd0d1Z7d2d3 Z8e e$d4dd5gd6d7 Z9e d8d9 Z:d:d; Z;d<d= Z<d>d? Z=d@dA Z>e$dBdCgdDd5fg dEdfgdFdG Z?dHdI Z@dJdK ZAdLdM ZBdNdO ZCdPdQ ZDdRdS ZEdTdU ZFdVdW ZGdXdY ZHdZd[ ZId\d] ZJd^d_ ZKd`da ZLdbdc ZMdddedfdgdhZNeOePePdidjdkZQdldm ZRdndo ZSddpdqZTdrds ZUdtdu ZVdvdw ZWdxdy ZXdzd{ ZYdd|d}ZZe"d~d Z[e"dd Z\dd Z]G dd deZ^G dd deZ_e$dde` ea gdd Zbdd Zcdd Zddd Zedd Zfdd Zgdd Zhdd Zidd Zjdd ZkdddZlejm$ddddd5ddddidgdd ZndS )z
Test the memory module.
    N)Memory)MemorizedFuncNotMemorizedFunc)MemorizedResultNotMemorizedResult)_FUNCTION_HASHES)register_store_backend_STORE_BACKENDS)_build_func_identifier_store_backend_factory)JobLibCollisionWarning)Paralleldelayed)StoreBackendBaseFileSystemStoreBackend)
with_numpynp)with_multiprocessing)parametrizeraiseswarns)hash   c             C   s   | d | S )z3 A module-level function for testing purposes.
        )xyr   r   I/var/www/html/venv/lib/python3.7/site-packages/joblib/test/test_memory.pyf)   s    r   c             C   sb   t |dd}|| } xFtdD ]:}x4tdD ](}| ||ksBtt||d ks.tq.W q W dS )z Given a function and an accumulator (a list that grows every
        time the function is called), check that the function can be
        decorated by memory to be a lazy identity.
    r   )locationverbose   r   r   N)r   cacherangeAssertionErrorlen)funcaccumulatorr   memoryi_r   r   r   check_identity_lazy1   s    
r+   c          	   C   s@   | j  \}tj|jd}t|d}|d W d Q R X d S )Nz
output.pklwgarbage)store_backend	get_itemsospathjoinopenwrite)r(   Zsingle_cache_itemZoutput_filenamer   r   r   r   corrupt_single_cache_item@   s    r5   c                s"   g   fdd}| | d|  S )Nc                s     |  d S )N)append)item)recordedr   r   append_to_recordN   s    z6monkeypatch_cached_func_warn.<locals>.append_to_recordwarn)setattr)r&   Zmonkeypatch_fixturer9   r   )r8   r   monkeypatch_cached_func_warnG   s    r<   c                s   t    fdd}t| | j xdD ]}x\dD ]T}t| jd||d}tj| jdd ||}|d	 |jd
d t }|d	}q0W t |d	 kst	|
|d	|kst	t |d	 ks&t	q&W d|_t| jdd}||d	 dS )z, Simple test of memory lazy evaluation.
    c                s     d | S )Nr   )r6   )l)r'   r   r   r   ^   s    
z"test_memory_integration.<locals>.f)FT)rN
   )r   r    	mmap_modecompressT)ignore_errorsr   F)r:   __main__r   )r   r    N)listr+   strpathr   shutilrmtreer"   clearr%   r$   eval
__module__)tmpdirr   rA   r@   r(   gcurrent_accumulatoroutr   )r'   r   test_memory_integrationV   s&    



rO   call_before_reducingTFc       
         s  xdD ]}d}d |}ttt||dd t d }d|_|ddksTt|jj	|ksdtt
| jd	d
}|| tt| d dkstt| d d	 }d|kst| d | }|d	krt|dgkstt|d g kst|r< d tt|d dksttdd fdddD }	n@tdd fdddD }	tt|d dkstt d tt|d dkstqtt|d dkst d tt|d dkstqW d S )N)r   r   z0
        def f(x):
            return x
        z<ipython-input-{}-000000000000>exec)filenamemoder   rC   r   r   )r   r    joblibzipython-inputr!   r   )n_jobsc             3   s   | ]}t  |V  qd S )N)r   ).0r)   )cached_fr   r   	<genexpr>   s    zHtest_parallel_call_cached_function_defined_in_jupyter.<locals>.<genexpr>)r   r   c             3   s   | ]}t  |V  qd S )N)r   )rV   r)   )rW   r   r   rX      s       )formatrQ   compiletextwrapdedentlocalsrJ   r$   __code__co_filenamer   rE   r"   r%   r0   listdirr   )
rK   rP   Z
session_noZipython_cell_sourceZipython_cell_idZ	aliased_fr(   Zf_cache_relative_directoryZf_cache_directoryresr   )rW   r   5test_parallel_call_cached_function_defined_in_jupyter   s@    




	rc   c                 sb   t    fdd} tddd}|| }x4tdD ](}t }|d t |d ks2tq2W dS )z, Test memory with location=None: no memoize c                s     d | S )Nr   )r6   )r=   )r'   r   r   ff   s    
ztest_no_memory.<locals>.ffNr   )r   r    rY   r   )rD   r   r"   r#   r%   r$   )rd   r(   Zggr*   rM   r   )r'   r   test_no_memory   s    
re   c                sR   t   d
 fdd	}t| | j t| jdd}||}|ddd	dksNtdS )z4 Test memory with a function with keyword arguments.Nr   c                s     d | S )Nr   )r6   )r=   m)r'   r   r   rL      s    
ztest_memory_kwarg.<locals>.gr   )r   r       r   )r=   rf   )Nr   )rD   r+   rE   r   r"   r$   )rK   rL   r(   r   )r'   r   test_memory_kwarg   s    
rh   c                s0   t    fddfdd}t| | j dS )z+ Test memory with a function with a lambda.c                s     d | S )z4 A helper function to define l as a lambda.
        r   )r6   )r   )r'   r   r   helper  s    
z"test_memory_lambda.<locals>.helperc                s    | S )Nr   )r   )ri   r   r   <lambda>      z$test_memory_lambda.<locals>.<lambda>N)rD   r+   rE   )rK   r=   r   )r'   ri   r   test_memory_lambda  s    rl   c          	   C   s   t | jdd}|jdd }|}|jdd }|}tt}|d |d W dQ R X t|dksftdt|d jks|tdS )	z> Check that name collisions with functions will raise warningsr   )r   r    c             S   s   | S )z0 A first function called name_collision
        r   )r   r   r   r   name_collision  s    z2test_memory_name_collision.<locals>.name_collisionc             S   s   | S )z1 A second function called name_collision
        r   )r   r   r   r   rm     s    r   N	collision)	r   rE   r"   r   r   r%   r$   strmessage)rK   r(   rm   abwarninfor   r   r   test_memory_name_collision  s    
rt   c          	   C   s   t | jdd}dd }||}dd }||}tt6}|ddksLt|ddks\t|ddksltW d Q R X t|dkstd S )	Nr   )r   r    c             S   s   | S )Nr   )r   r   r   r   rj   2  rk   z7test_memory_warning_lambda_collisions.<locals>.<lambda>c             S   s   | d S )Nr   r   )r   r   r   r   rj   4  rk   r   r   rY   )r   rE   r"   r   r   r$   r%   )rK   r(   rq   rr   rs   r   r   r   %test_memory_warning_lambda_collisions/  s    


ru   c          	   C   s   t | jdd}td}||}td}||}tt}|d |d |d W d Q R X t|dksntdt|d j	
 kstd S )Nr   )r   r    zlambda x: xzlambda x: x+1r   r   zcannot detect)r   rE   rI   r"   r   r   r%   r$   ro   rp   lower)rK   r(   Za1b1rs   r   r   r   'test_memory_warning_collision_detectionA  s    


rx   c                s8   t    fdd}ddl}||d}t| | j dS )z$ Test memory with functools.partial.c                s     d |S )z4 A helper function to define l as a lambda.
        r   )r6   )r   r   )r'   r   r   r&   W  s    
z!test_memory_partial.<locals>.funcr   Nr   )rD   	functoolspartialr+   rE   )rK   r&   ry   functionr   )r'   r   test_memory_partialS  s
    r|   c             C   s4   t | jdd}td}||}|ddks0tdS )zF Smoke test memory with a function with a function defined in an eval.r   )r   r    zlambda x: xr   N)r   rE   rI   r"   r$   )rK   r(   rf   mmr   r   r   test_memory_evalc  s    
r~   c             C   s   t | }| d |S )zx A function with a side effect in its arguments.

        Return the length of its argument and append one element.
    N)r%   r6   )r   Zlen_xr   r   r   count_and_appendm  s    
r   c             C   s8   t | jdd}|t}| dks&t| dks4tdS )zx Check that if a function has a side effect in its arguments, it
        should use the hash of changing arguments.
    r   )r   r    r   N)r   rE   r"   r   r$   )rK   r(   r&   r   r   r   test_argument_changew  s    
r   r@   r>   c       	         s   t   d	 fdd	}t| j|dd}||}tjd}xVtdD ]J}|d}x:tdD ].}t	|||ksvt
t |d ks\t
q\W qDW dS )
z/ Test memory with a function with numpy arrays.Nc                s     d | S )Nr   )r6   )r=   )r'   r   r   n  s    
ztest_memory_numpy.<locals>.nr   )r   r@   r    r!   )r?   r?   r   )N)rD   r   rE   r"   r   randomZRandomStater#   Zrandom_sampleallr$   r%   )	rK   r@   r   r(   Zcached_nZrndr)   rq   r*   r   )r'   r   test_memory_numpy  s    

r   c       
      C   s   t | jddd}| dd }td}||}||}t|tjsJt|jdksXtt|tjsht|jdksvt~~t	
  t| t||}||}t|dkstd}	|	|d kstt|tjst|jdkstd	S )
z8Check that mmap_mode is respected even at the first callr>   r   )r   r@   r    c             S   s   | d S )Nr   r   )rq   r   r   r   twice  s    z0test_memory_numpy_check_mmap_mode.<locals>.twicer!   r   zException while loading resultsN)r   rE   r"   r   Zones
isinstanceZmemmapr$   rS   gcZcollectr5   r<   r%   )
rK   monkeypatchr(   r   rq   rr   crecorded_warningsdexception_msgr   r   r   !test_memory_numpy_check_mmap_mode  s(    

r   c          
      sj   t | jdd}G dd dt |jd
 fdd	}|  x,tdD ] }t  |d W d	Q R X qBW d	S )z1 Smoketest the exception handling of Memory.
    r   )r   r    c               @   s   e Zd ZdS )z*test_memory_exception.<locals>.MyExceptionN)__name__rJ   __qualname__r   r   r   r   MyException  s   r   c                s   | r d S )Nr   )exc)r   r   r   h  s    z test_memory_exception.<locals>.hr!   r   N)r   )r   rE   	Exceptionr"   r#   r   )rK   r(   r   r*   r   )r   r   test_memory_exception  s    
r   c                s   t | jdd}t  |jdgdd fdd	}|jdgks@t|ddd t dks\t|ddd t dksxt|dd	d t dkstd
S )z# Test the ignore feature of memory r   )r   r    r   )ignorer   c                s     d d S )Nr   )r6   )r   r   )r'   r   r   z  s    ztest_memory_ignore.<locals>.z)r   r   N)r   )r   rE   rD   r"   r   r$   r%   )rK   r(   r   r   )r'   r   test_memory_ignore  s    r   c                s   t | jdd}t  dd }|jdgd|d fdd		}|jdgksLt|ddd
 t dksht|ddd
 t dkst|ddd
 t dkstdS )z; Test the ignore feature of memory on a decorated function r   )r   r    c                s   t   fdd}|S )Nc                 s
    | |S )Nr   )argskwargs)r   r   r   wrapped  s    z?test_memory_ignore_decorated.<locals>.decorate.<locals>.wrapped)ry   wraps)r   r   r   )r   r   decorate  s    z.test_memory_ignore_decorated.<locals>.decorater   )r   r   c                s     d d S )Nr   )r6   )r   r   )r'   r   r   r     s    z'test_memory_ignore_decorated.<locals>.z)r   r   N)r   )r   rE   rD   r"   r   r$   r%   )rK   r(   r   r   r   )r'   r   test_memory_ignore_decorated  s    r   c             C   sT   t | jdd}|jdd }|ddks,t|dddks>t|dddksPtd	S )
z_Non-regression test against 0.12.0 changes.

    https://github.com/joblib/joblib/pull/751
    r   )r   r    c             S   s   | d S )Nr   r   )rq   r   r   r   plus_one  s    z,test_memory_args_as_kwargs.<locals>.plus_oner   r   )rq   r!   N)r   rE   r"   r$   )rK   r(   r   r   r   r   test_memory_args_as_kwargs  s
    r   zignore, verbose, mmap_moder   d   r?   c             C   sT   t | jdd}|j|||ddd }|j|ks4t|j|ksBt|j|ksPtdS )z7Check cache may be called with kwargs before decoratingr   )r   r    )r   r    r@   c             S   s   d S )Nr   )r   r   r   r   r      s    z"test_partial_decoration.<locals>.zN)r   rE   r"   r   r$   Z_verboser@   )rK   r   r    r@   r(   r   r   r   r   test_partial_decoration  s
    r   c       	      C   s,  t | jdd}td}|d | jd	| j}|t}tt}t	j
|jj|}||ksbtt	j
|srt|jt	j
|jjkstt  | rtt	j
t	j
|dst| st|d\}}t	j
|jj||}|d}t	j
|stt	t	j
|d ||dks(td S )
Nr   )r   r    .r   rT   zfunc_code.pyr   z
output.pkl)rT   )r   rE   r   splitr6   r2   r"   r   r
   r0   r1   r.   r   r$   existsdirnamer   rH   _check_previous_func_code_get_output_identifiersremove)	rK   r(   r1   rL   func_idr   args_id
output_dirrq   r   r   r   test_func_dir)  s(    


r   c       
      C   s   t | jdd}|t}|d}tt|}|d\}}tj	
|jj||}tj	|sbt||j||gkszttt|}|jj|jjkstt d dd}tt| |t}tt|}	|	d d S )Nr   )r   r    r   )r   rE   r"   r   pickleloadsdumpsr   r0   r1   r2   r.   r   r   r$   Z	load_item)
rK   r(   rL   outputr   r   r   r   Zmemory2gpr   r   r   test_persistenceI  s    

r   c             C   s   xt t| jt| jddtfD ]\}|d}|r8tt|tsFt|ddksVt|d}|shtt|tsvt|	  q"W d S )Nr   )r   r    r      )
r   r   rE   r   r"   Zcheck_call_in_cacher$   r   boolrH   )rK   r&   resultr   r   r   test_check_call_in_cache`  s    

r   c          
   C   s   xt tt| jttt| jddttd dtfttttfD ]l\}}|ddks\t	|
d}t||stt	| dkst	|  tt |  W d Q R X |  qDW d S )Nr   )r   r    )r   r   r   )zipr   r   rE   r   r   r"   r   r   r$   call_and_shelver   getrH   r   KeyError)rK   r&   Resultr   r   r   r   test_call_and_shelvem  s    

r   c          	   C   sz   t | jddt}|d}t|ts,ttt	}|j
|jksFtW d Q R X t|dks`tdt|d jksvtd S )Nr   )r   r    r   r   z1The 'argument_hash' attribute has been deprecated)r   rE   r"   r   r   r   r   r$   r   DeprecationWarningargument_hashr   r%   ro   rp   )rK   r&   r   r,   r   r   r   "test_call_and_shelve_argument_hash  s    

r   c       
      C   s  |  d}|d t|jj}td | dks<t	|t|jjkrXt
d t| jdd}|t}|d\}}tj |jj||d}|ddkst	t|j}td	 |d}	t|	tst	t|j|kst	td	 |	 dkst	t|j|kst	d
S )z6Check call_and_shelve only load stored data if needed.Ztest_accessg      ?z>filesystem does not support fine-grained access time attributer   )r   r    r   z
output.pklr   r   N)r2   r4   r0   statrE   st_atimetimesleepreadr$   pytestskipr   r"   r   r   r1   r.   r   r   r   r   r   )
rK   Ztest_access_time_fileZtest_access_timer(   r&   r   r   result_pathZfirst_access_timer   r   r   r   .test_call_and_shelve_lazily_load_stored_result  s*    








r   c          
   C   s   xt t| jttfD ]z}| dj}|d}t|d}t|| W d Q R X t|d}t	|}W d Q R X |
 |
 kstt| qW d S )Nzpickling_test.datr   wbrb)r   r   rE   r   r2   r   r3   r   dumploadr   r$   r0   r   )rK   r&   rR   r   fpresult2r   r   r   test_memorized_pickling  s    
r   c             C   s
  t t| j}|d}t t| j}|d}| | ks@tt|t|ksTttt}t| t|d t t| jdt d}|d}|  t t| jdd}|d}|  t t| jdt d}|d}|  t t| jdd}|d}|  d S )Nr      )r    	timestamp)r    r   )	r   r   rE   r   r   r$   reprr   r   )rK   r&   r   Zfunc2r   r   r   r   test_memorized_repr  s*    





r   c          	   C   sX  | dj}tj|d}d}t|d}|| W d Q R X || dd l}t	|jdd}|
|j}	|	d |	d |	d t|d}|d	|  W d Q R X |	d |	d t| t | d
}t|d}|| W d Q R X |	d |	d tjd tjd dd l}|
|j}	|	d |	d |  \}
}|
dksTtd S )NZ
tmp_importztmp_joblib_.pyz$def f(x):
    print(x)
    return x
r,   r   )r   r    r   r   z

z-def f(x):
    print("x=%s" % x)
    return x
z
Reloading
tmp_joblib_z1
2
Reloading
x=1
)mkdirrE   r0   r1   r2   r3   r4   Zsyspath_prependr   r   r"   r   rF   rG   sysstdoutmodulespop
readouterrr$   )ZcapsysrK   r   dir_namerR   contentZmodule_filetmpr(   r   rN   errr   r   r   test_memory_file_modification  s>    


r   c             C   s   d S )Nr   )rq   rr   r   r   r   _function_to_cache  s    r   c             C   s   | | S )Nr   )rq   rr   r   r   r   _sum  s    r   c             C   s   | | S )Nr   )rq   rr   r   r   r   _product!  s    r   c          	   C   s   t jt_t| jdd}|t}|dddks2t|dddksDttt2 t	jt_|dddksht|dddksztW d Q R X d S )Nr   )r   r    r   r   r!   )
r   r_   r   r   rE   r"   r$   r   r   r   )rK   r(   r   r   r   r   *test_memory_in_memory_function_code_change%  s    

r   c              C   s   t d d} |   d S )N)r   )r   rH   )r(   r   r   r   $test_clear_memory_with_none_location5  s    
r   kw1kw2)r   r   c            C   s   | |||fS )Nr   )rq   rr   r   r   r   r   r   func_with_kwonly_args:  s    r   )rq   rr   returnc             C   s   | | S )Nr   )rq   rr   r   r   r   func_with_signature>  s    r   c          	   C   s   t | jdd}|t}|dddddks.ttt}|ddddd	 W d Q R X |d
 |ddddd tt}|ddddd	 W d Q R X |d
 |jtdgd}|ddddddkst|ddddddkstd S )Nr   )r   r    r   r   r!   )r   )r   r   r!   r   rY   )r   z?Keyword-only parameter 'kw1' was passed as positional parameter)r   r   r   )r   )r   r   r!   rY   Zignored)r   rE   r"   r   r$   r   
ValueErrormatch)rK   r(   func_cachedexcinfor   r   r   !test_memory_func_with_kwonly_argsB  s    




r   c             C   s.   t | jdd}|t}|dddks*td S )Nr   )r   r    r   g       @g      @)r   rE   r"   r   r$   )rK   r(   r   r   r   r   test_memory_func_with_signature^  s    
r   c                sx   t | jdd}| dd tt|}x|D ]}| q0W t fdd|D } fdd|D }||fS )Nr   )r   r    c             S   s   dS )N  aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaar   )argr   r   r   get_1000_bytesh  s    z(_setup_toy_cache.<locals>.get_1000_bytesc                s   g | ]}  |d  qS )r   )r   )rV   r   )r   r   r   
<listcomp>q  s   z$_setup_toy_cache.<locals>.<listcomp>c                s    g | ]}t jjj |qS r   )r0   r1   r2   r.   r   )rV   r   )r   r   r   r   r   t  s   )r   rE   r"   rD   r#   r
   )rK   Z
num_inputsr(   inputsr   Zhash_dirnamesZfull_hashdirsr   )r   r   r   _setup_toy_cachee  s    

r   c                s   t | \}}}|j }dd |D }t|t|ks:tdd   fdd|D }dd |D }||ksntdd |D }dd |D }	d	d |D }
|
|	kstd S )
Nc             S   s   g | ]
}|j qS r   )r1   )rV   cir   r   r   r   }  s    z#test__get_items.<locals>.<listcomp>c                s*    fddt  D }tdd |D S )Nc                s   g | ]}t j |qS r   )r0   r1   r2   )rV   fn)	directoryr   r   r     s   z;test__get_items.<locals>.get_files_size.<locals>.<listcomp>c             s   s   | ]}t j|V  qd S )N)r0   r1   getsize)rV   r   r   r   r   rX     s    z:test__get_items.<locals>.get_files_size.<locals>.<genexpr>)r0   ra   sum)r   Z
full_pathsr   )r   r   get_files_size  s    
z'test__get_items.<locals>.get_files_sizec                s   g | ]} |qS r   r   )rV   hash_dir)r   r   r   r     s   c             S   s   g | ]
}|j qS r   )size)rV   r   r   r   r   r     s    c             S   s   g | ]}t j|d qS )z
output.pkl)r0   r1   r2   )rV   r   r   r   r   r     s   c             S   s    g | ]}t j tj|qS r   )datetimefromtimestampr0   r1   getatime)rV   r   r   r   r   r     s   c             S   s   g | ]
}|j qS r   )last_access)rV   r   r   r   r   r     s    )r   r.   r/   setr$   )rK   r(   Zexpected_hash_dirsr*   itemsZ	hash_dirsZexpected_hash_cache_sizesZhash_cache_sizesZoutput_filenamesZexpected_last_accessesZlast_accessesr   )r   r   test__get_itemsz  s    

r  c             C   s   t | \}}}|j }|jd}t|}tt|t|sDtt||d ksXt|jd}t|t|ksxt|jd}|g kstd}	|j|	}
t|
stt||jd}t|	|}t
dd |D td	d |D kstd S )
NZ2Kr   i   1Mi  ip  c             s   s   | ]}|j V  qd S )N)r  )rV   r   r   r   r   rX     s    z,test__get_items_to_delete.<locals>.<genexpr>c             s   s   | ]}|j V  qd S )N)r  )rV   r   r   r   r   rX     s    )r   r.   r/   Z_get_items_to_deleter%   r  issubsetr$   sorted
differencemaxmin)rK   r(   Zexpected_hash_cachedirsr*   r  Zitems_to_deleteZ	nb_hashesZitems_to_delete_2048bZitems_to_delete_emptybytes_limit_too_smallZitems_to_delete_500bZitems_to_delete_6000bZsurviving_itemsr   r   r   test__get_items_to_delete  s$    
r  c             C   s   t | \}}}|j }|  |j }t|t|ks>td|_|  |j }t|t|ksjtd|_|  |j }tt|t|stt	|dkstd}||_|  |j }|g kstd S )Nr  Z3Kr   i  )
r   r.   r/   Zreduce_sizer  r$   bytes_limitr  r  r%   )rK   r(   r*   Zref_cache_itemsZcache_itemsr  r   r   r   test_memory_reduce_size  s&    




r  c             C   s@   t | \}}}|  t|jjg ks,t|jddr<td S )NrY   )
stacklevel)r   rH   r0   ra   r.   r   r$   r   )rK   r(   r*   rL   r   r   r   test_memory_clear  s    r  c              C   s   dgd } | S )Nr   i  r   )complex_objr   r   r   fast_func_with_complex_output  s    
r  c             C   s"   dd t tdD }| r|S dS )Nc             S   s   i | ]}|t |qS r   )ro   )rV   r)   r   r   r   
<dictcomp>  s    z=fast_func_with_conditional_complex_output.<locals>.<dictcomp>g     j@zsimple output)r#   int)Zcomplex_outputr  r   r   r   )fast_func_with_conditional_complex_output  s    r  c                sb   t | jd}|t tdd fddtdD  | \}}d}||ksRt||ks^td S )N)r   r   )rU   c             3   s   | ]}t   V  qd S )N)r   )rV   r)   )r   r   r   rX     s    zMtest_cached_function_race_condition_when_persisting_output.<locals>.<genexpr>r!   zException while loading results)r   rE   r"   r  r   r#   r   r$   )rK   capfdr(   r   stderrr   r   )r   r   :test_cached_function_race_condition_when_persisting_output  s    
 r  c                sb   t | jd}|t tdd fddtdD  | \}}d}||ksRt||ks^td S )N)r   r   )rU   c             3   s*   | ]"}t  |d  dkrdndV  qdS )r   r   TFN)r   )rV   r)   )r   r   r   rX     s   zOtest_cached_function_race_condition_when_persisting_output_2.<locals>.<genexpr>r!   zException while loading results)r   rE   r"   r  r   r#   r   r$   )rK   r  r(   r   r  r   r   )r   r   <test_cached_function_race_condition_when_persisting_output_2  s    
r  c          
   C   s   t | jd}dd }||}d}||\}}||ks:tt| t||}||\}	}
t|dkshtd}||d ks|t|	|kst|
|kstt| ||}y|  tdW n8 t	k
r } zd	}|t
|jkstW d d }~X Y nX d S )
N)r   c             S   s   t d | t   fS )Ng{Gz?)r   r   )r   r   r   r   r&     s    
zItest_memory_recomputes_after_an_error_while_loading_results.<locals>.funcr   r   zException while loading resultsr   z<It normally not possible to load a corrupted MemorizedResultzis corrupted)r   rE   r"   r$   r5   r<   r%   r   r   r   ro   r   )rK   r   r(   r&   Zcached_funcZ	input_argr   r   r   Zrecomputed_argZrecomputed_timestampr   	referenceerp   r   r   r   ;test_memory_recomputes_after_an_error_while_loading_results  s.    



r  c               @   s   e Zd ZdZdS )IncompleteStoreBackendzAThis backend cannot be instantiated and should raise a TypeError.N)r   rJ   r   __doc__r   r   r   r   r  C  s   r  c               @   sP   e Zd ZdZdd Zdd Zdd Zdd	 Zd
d Zdd Z	dd Z
dd ZdS )DummyStoreBackendz(A dummy store backend that does nothing.c             O   s   dS )zOpen an item on store.Nr   )selfr   r   r   r   r   
_open_itemK  s    zDummyStoreBackend._open_itemc             C   s   dS )z!Check if an item location exists.Nr   )r!  r   r   r   r   _item_existsO  s    zDummyStoreBackend._item_existsc             C   s   dS )z&Move an item from src to dst in store.Nr   )r!  srcdstr   r   r   
_move_itemS  s    zDummyStoreBackend._move_itemc             C   s   dS )zCreate location on store.Nr   )r!  r   r   r   r   create_locationW  s    z!DummyStoreBackend.create_locationc             C   s   dS )z&Check if an object exists in the storeFr   )r!  objr   r   r   r   [  s    zDummyStoreBackend.existsc             C   s   dS )zClear object on storeNr   )r!  r(  r   r   r   clear_location_  s    z DummyStoreBackend.clear_locationc             C   s   g S )z3Returns the whole list of items available in cache.r   )r!  r   r   r   r/   c  s    zDummyStoreBackend.get_itemsc             O   s   dS )zConfigure the storeNr   )r!  r   r   r   r   r   r   	configureg  s    zDummyStoreBackend.configureN)r   rJ   r   r  r"  r#  r&  r'  r   r)  r/   r*  r   r   r   r   r   H  s   r   invalid_prefixc          	   C   s,   t t}t| d  W d Q R X |d d S )Nz&Store backend name should be a string*)r   r   r   r   )r+  r   r   r   r   (test_register_invalid_store_backends_keyl  s    
r,  c           	   C   s,   t t} tdd  W d Q R X | d d S )Nfsz.Store backend should inherit StoreBackendBase*)r   r   r   r   )r   r   r   r   +test_register_invalid_store_backends_objectt  s    
r.  c           	   C   s.   t t} tddd W d Q R X | d d S )Nz/tmp/joblibunknown)r   backendzUnknown location*)r   	TypeErrorr   r   )r   r   r   r   !test_memory_default_store_backend|  s    
r2  c           	   C   sR   G dd d} |  }t t}td|d W d Q R X d}|t|d jksNtd S )Nc               @   s   e Zd ZdS )zHtest_warning_on_unknown_location_type.<locals>.NonSupportedLocationClassN)r   rJ   r   r   r   r   r   NonSupportedLocationClass  s   r3  local)r   zbInstantiating a backend using a NonSupportedLocationClass as a location is not supported by joblibr   )r   UserWarningr   ro   rp   r$   )r3  Zunsupported_locationrs   Zexpected_mesager   r   r   %test_warning_on_unknown_location_type  s    
r6  c           	   C   sN   d} t | t | tft ks"ttt}t| d W d Q R X |d d S )NZisbZfake_locationzNCan't instantiate abstract class IncompleteStoreBackend with abstract methods*)	r   r  r	   r  r$   r   r1  r   r   )backend_namer   r   r   r   )test_instanciate_incomplete_store_backend  s    

r8  c              C   s>   d} t | t | tft ks"tt| d}t|ts:td S )NZdsbZdummy_location)r   r   r	   r  r$   r   r   )r7  backend_objr   r   r   test_dummy_store_backend  s
    

r:  c              C   s&   t d} td| }|jdks"td S )NZsome_folderr4  )pathlibPathr   r   r$   )r1   r9  r   r   r   0test_instanciate_store_backend_with_pathlib_path  s    

r=  c             C   sj   d}t  }|jd kstt| t||jd dks8t|| j t||j| jdks^tt| d S )Nz-FileSystemStoreBackend(location="{location}"))r   )r   r   r$   r   ro   rZ   r*  rE   )rK   Zrepr_patternr0  r   r   r   "test_filesystem_store_backend_repr  s    r>  c             C   s   dd }t | jdd}||}d}t||j||jjdksBt|dd}d}t||j|jj|j	|j
d	ksvtt|d
j|jjdkstd S )Nc             S   s   | | S )Nr   )rq   rr   r   r   r   my_func  s    z)test_memory_objects_repr.<locals>.my_funcr   )r   r    z/MemorizedFunc(func={func}, location={location}))r&   r   *   zJMemorizedResult(location="{location}", func="{func}", args_id="{args_id}"))r   r&   r   zMemory(location={location}))r   )r   rE   r"   ro   rZ   r.   r   r$   r   r   r   )rK   r?  r(   Zmemorized_funcZmemorized_func_reprmemorized_resultZmemorized_result_reprr   r   r   test_memory_objects_repr  s    



rB  c             C   s   t | jd}|jdd }|d}t|}t|}|jj|jjksLt	|j
|j
ks\t	|j|jkslt	t|t|kst	d S )N)r   c             S   s   | d S )Nr   r   )r   r   r   r   rL     s    z'test_memorized_result_pickle.<locals>.grY   )r   rE   r"   r   r   r   r   r.   r   r$   r&   r   ro   )rK   r(   rL   rA  Zmemorized_result_pickleZmemorized_result_loadsr   r   r   test_memorized_result_pickle  s    


rC  c             C   sl   |d krg }t | }t |}t| t| ks8tx.| D ]"}||krPqB|| || ksBtqBW d S )N)varsr  keysr$   )leftrightignored_attrsZ	left_varsZ
right_varsattrr   r   r   compare  s    rJ  memory_kwargsr!   r   )rA   r    r   g    .AZ	parameterZunused)r@   r    r  Zbackend_optionsc             C   s  t f d| ji|}tt|}t|j|j t||tdddgd t|t|ks^t	|
t}tt|}t|j|j t||tdddgd t|t|kst	|d}tt|}t|j|j t||tdddgd t|t|kst	d S )Nr   r.   r   Z_func_code_id)rH  r   )r   rE   r   r   r   rJ  r.   r  r   r$   r"   r   r   )rK   rK  r(   Zmemory_reloadedr   Zfunc_cached_reloadedrA  Zmemorized_result_reloadedr   r   r   test_memory_pickle_dump_load  s&    

rL  )r   )r?   )T)N)or  ry   r   rF   r0   os.pathr;  r   r   r   r   r\   r   Zjoblib.memoryr   r   r   r   r   r   r   r	   r
   r   r   Zjoblib.parallelr   r   Zjoblib._store_backendsr   r   Zjoblib.test.commonr   r   r   Zjoblib.testingr   r   r   Zjoblib.hashingr   r   r+   r5   r<   rO   rc   re   rh   rl   rt   ru   rx   r|   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  floatr   r   r   r   r  r  r  r  r  r  r  r  r  r  r   dictrD   r,  r.  r2  r6  r8  r:  r=  r>  rB  rC  rJ  markrL  r   r   r   r   <module>   s   
+c

( $ 8
!

/$
