B Y0dz @sdZddlmZddlZddlZddlmZddlZddlZddl m Z m Z m Z m Z mZddlZddlZddlZddlmZmZmZmZmZddlmZmZmZmZmZddlZddl Z ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'dd l(m)Z)m*Z*dd l+m,Z,dd l-m.Z.e*Z/e0eeeZ1e12d ej3Gd ddZ4ej3GdddZ5ddddZ6dddddZ7ddddZ8dQdddddd Z9d!d"Z:d#dd$d%d&Z;dRd#d)d*d)d+dd,d-d.Zd*d7d8d9d:Z?d#d;d;d<d=d>Z@dSd#d)d;d*ddd;d+dd@ dAdBZAGdCdDdDe jBe ZCGdEdFdFejDZEdGdd)d)d;ddHdIdJdKZFd#dddLdMZGd#d)ddNdOdPZHdS)TzCommon IO api utilities) annotationsN)abc)BufferedIOBaseBytesIO RawIOBaseStringIO TextIOWrapper)IOAnyAnyStrMappingcast)urljoinurlparse uses_netloc uses_params uses_relative)BufferCompressionDictCompressionOptions FileOrBufferFilePathOrBufferStorageOptions) get_lzma_file import_lzma)import_optional_dependency) is_file_likec@s>eZdZUdZded<ded<ded<ded<d Zd ed <d S) IOArgsa) Return value of io/common.py:_get_filepath_or_buffer. Note (copy&past from io/parsers): filepath_or_buffer can be Union[FilePathOrBuffer, s3fs.S3File, gcsfs.GCSFile] though mypy handling of conditional imports is difficult. See https://github.com/python/mypy/issues/1297 rfilepath_or_bufferstrencodingmoder compressionFbool should_closeN)__name__ __module__ __qualname____doc____annotations__r%r+r+B/var/www/html/venv/lib/python3.7/site-packages/pandas/io/common.pyr;s rc@szeZdZUdZded<ded<ejedZded<d Z d ed <d Z d ed <d dddZ ddddZ dd dddZ dS) IOHandlesau Return value of io/common.py:get_handle Can be used as a context manager. This is used to easily close created buffers and to handle corner cases when TextIOWrapper is inserted. handle: The file handle to be used. created_handles: All file handles that are created by get_handle is_wrapped: Whether a TextIOWrapper needs to be detached. rhandlerr#)default_factoryz list[Buffer]created_handlesFr$ is_wrappedis_mmapNone)returnc Cs~|jr8t|jtst|j|j|j|jyx|jD] }| qBWWnt t fk rlYnXg|_d|_dS)z Close all created buffers. Note: If a TextIOWrapper was inserted, it is flushed and detached to avoid closing the potentially user-created buffer. FN) r1 isinstancer.rAssertionErrorflushdetachr0removecloseOSError ValueError)selfr.r+r+r,r:bs   zIOHandles.closecCs|S)Nr+)r=r+r+r, __enter__vszIOHandles.__enter__r )argsr4cGs |dS)N)r:)r=r?r+r+r,__exit__yszIOHandles.__exit__N)r&r'r(r)r* dataclassesfieldlistr0r1r2r:r>r@r+r+r+r,r-Ms   r-r$)r4cCst|tsdSt|jtkS)z Check to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If `url` has a valid protocol return True otherwise False. F)r5r parse_urlscheme _VALID_URLS)urlr+r+r,is_url}s rHzFileOrBuffer[AnyStr])rr4cCst|trtj|S|S)a] Return the argument with an initial component of ~ or ~user replaced by that user's home directory. Parameters ---------- filepath_or_buffer : object to be converted if possible Returns ------- expanded_filepath_or_buffer : an expanded filepath or the input if not expandable )r5r ospath expanduser)rr+r+r, _expand_users  rLr3cCst|trtddS)NzPassing a bool to header is invalid. Use header=None for no header or header=int or list-like of ints to specify the row(s) making up the column names)r5r$ TypeError)headerr+r+r,validate_header_args rOFzFilePathOrBuffer[AnyStr])rconvert_file_liker4cCs6|st|rttt|St|tjr.|}t|S)a Attempt to convert a path-like object to a string. Parameters ---------- filepath_or_buffer : object to be converted Returns ------- str_filepath_or_buffer : maybe a string version of the object Notes ----- Objects supporting the fspath protocol (python 3.6+) are coerced according to its __fspath__ method. Any other object is passed through unchanged, which includes bytes, strings, buffers, or anything else that's not even path-like. ) rr rr r5rIPathLike __fspath__rL)rrPr+r+r,stringify_paths   rScOsddl}|jj||S)z` Lazy-import wrapper for stdlib urlopen, as that imports a big chunk of the stdlib. rN)urllib.requestrequesturlopen)r?kwargsurllibr+r+r,rVsrVr)rGr4cCst|tod|ko|d S)zR Returns true if the given URL looks like something fsspec can handle z://)zhttp://zhttps://)r5r startswith)rGr+r+r, is_fsspec_urls rZutf-8rr rr)rr!r#r"storage_optionsr4c Cst|}t|\}}t||}|rHt|drHd|krHtjdtddd}t||d}|dk rl|dd  }d |kr|d kr|d krt|d |t |}d|krd|kr|d7}t |t r>t |r>|pi}ddl}|jj||d}t|2} | jdd} | dkrddi}t| } WdQRXt| ||d|dSt|rht |t sXt|drp|dd}|dr|dd}td} g} y&tdddlm}m}||tg} Wntk rYnXy$| j |fd|i|pi }Wn^t!| k rT|dkr ddi}nt|}d|d<| j |fd|i|pFi }YnXt|||d|dS|rvt"dt |t t#t$j$frtt%|||d|dSt&|sd t'|}t"|t|||d|dS)!a If the filepath_or_buffer is a url, translate and return the buffer. Otherwise passthrough. Parameters ---------- filepath_or_buffer : a url, filepath (str, py.path.local or pathlib.Path), or buffer compression : {{'gzip', 'bz2', 'zip', 'xz', None}}, optional encoding : the encoding to use to decode bytes, default is 'utf-8' mode : str, optional storage_options : dict, optional Extra options that make sense for a particular storage connection, e.g. host, port, username, password, etc., if using a URL that will be parsed by ``fsspec``, e.g., starting "s3://", "gcs://". An error will be raised if providing this argument with a local path or a file-like buffer. See the fsspec and backend storage implementation docs for the set of allowed keys and values .. versionadded:: 1.2.0 ..versionchange:: 1.2.0 Returns the dataclass IOArgs. writebzDcompression has no effect when passing a non-binary object as input.) stacklevelN)method_-w)bz2xz)zutf-16zutf-32z( will not write the byte order mark for tr)headerszContent-EncodinggziprbT)rr!r#r%r"zs3a://zs3://zs3n://fsspecZbotocore) ClientErrorNoCredentialsErrorr"Zanonz?storage_options passed with file object or non-fsspec file pathFz)Invalid file path or buffer object type: )(rSget_compression_methodinfer_compressionhasattrwarningswarnRuntimeWarningdictreplacelowerUnicodeWarningr5r rHrTrURequestrVrigetrreadrrZr6rYrZbotocore.exceptionsrlrmPermissionError ImportErroropentupler<bytesmmaprLrtype)rr!r#r"r]compression_methodZ fsspec_moderXZreq_inforeqcontent_encodingreaderrkZerr_types_to_retry_with_anonrlrmZfile_objmsgr+r+r,_get_filepath_or_buffers!               " r)rJr4cCsddlm}td||S)z converts an absolute native path to a FILE URL. Parameters ---------- path : a path in native format Returns ------- a valid FILE URL r) pathname2urlzfile:)rTrr)rJrr+r+r,file_path_to_urls rz.gzz.bz2z.zipz.xz)rjrfziprgz"tuple[str | None, CompressionDict])r#r4c Cs`t|trPt|}y|d}WqXtk rL}ztd|Wdd}~XYqXXni}|}||fS)a Simplifies a compression argument to a compression method string and a mapping containing additional arguments. Parameters ---------- compression : str or mapping If string, specifies the compression method. If mapping, value at key 'method' specifies compression method. Returns ------- tuple of ({compression method}, Optional[str] {compression arguments}, Dict[str, Any]) Raises ------ ValueError on mapping missing 'method' key rbz.If mapping, compression must have key 'method'N)r5r rtpopKeyErrorr<)r#compression_argsrerrr+r+r,rns rnz str | None)rr#r4cCs|dkr dS|dkrZt|dd}t|ts.dSx&tD]\}}||r8|Sq8WdS|tkrf|Sddgtt}d|d|}t|dS)a Get the compression method for filepath_or_buffer. If compression='infer', the inferred compression method is returned. Otherwise, the input compression method is returned unchanged, unless it's invalid, in which case an error is raised. Parameters ---------- filepath_or_buffer : str or file handle File path or object. compression : {'infer', 'gzip', 'bz2', 'zip', 'xz', None} If 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). Returns ------- string or None Raises ------ ValueError on invalid compression specified. NZinferT)rPzUnrecognized compression type: z Valid compression types are ) rSr5r _compression_to_extensionitemsrvendswithsortedr<)rr# extensionZvalidrr+r+r,ros   roT) path_or_bufr"r!r# memory_mapis_texterrorsr]r4cCs|pd}t||r"d|kr"|d7}t|tr4|}|dkrTtd|dddt|||||d}|j} t| ||j|j ||j d t k\} }} t| t} t |j } | d }|r|j d d |_ |d kr| rt| tsttjf| |j d | } ntjf| |j d| } n|dkr:tj| fd|j i| } n|dkrt| |j f| } | j dkr| | | } t| dkr| | } n,t| dkrtd|ntd| n.|dkrtt| |j } nd|}t|t| trt| | nNt| trV|jr@d|j kr@t| |j |j|d d} n t| |j } | | d}|r|stt| |j rt| |j|d d} | | t|jtp|j }| |jrt|jtrt| |jt| trtt| | |||j dS)a Get file handle for given path/buffer and mode. Parameters ---------- path_or_buf : str or file handle File path or object. mode : str Mode to open path_or_buf with. encoding : str or None Encoding to use. compression : str or dict, default None If string, specifies compression mode. If dict, value at key 'method' specifies compression mode. Compression mode must be one of {'infer', 'gzip', 'bz2', 'zip', 'xz', None}. If compression mode is 'infer' and `filepath_or_buffer` is path-like, then detect compression from the following extensions: '.gz', '.bz2', '.zip', or '.xz' (otherwise no compression). If dict and compression mode is one of {'zip', 'gzip', 'bz2'}, or inferred as one of the above, other entries passed as additional compression options. .. versionchanged:: 1.0.0 May now be a dict with key 'method' as compression mode and other keys as compression options if compression mode is 'zip'. .. versionchanged:: 1.1.0 Passing compression options as keys in dict is now supported for compression modes 'gzip' and 'bz2' as well as 'zip'. memory_map : bool, default False See parsers._parser_params for more information. is_text : bool, default True Whether the type of the content passed to the file/buffer is string or bytes. This is not the same as `"b" not in mode`. If a string content is passed to a binary file/buffer, a wrapper is inserted. errors : str, default 'strict' Specifies how encoding and decoding errors are to be handled. See the errors argument for :func:`open` for a full list of options. storage_options: StorageOptions = None Passed to _get_filepath_or_buffer .. versionchanged:: 1.2.0 Returns the dataclass IOHandles zutf-8r_) Nstrictignoreruxmlcharrefreplacebackslashreplace namereplacesurrogateescape surrogatepassz%Invalid value for `encoding_errors` (z). Please see z=https://docs.python.org/3/library/codecs.html#error-handlers zfor valid values.)r!r#r"r]rbrhrrj)filenamer")fileobjr"rfr"rr\rzZero files found in ZIP file z9Multiple files found in ZIP file. Only one file per ZIP: rgzUnrecognized compression type: )r!rnewlineF)r.r0r1r2r#)_is_binary_moder5r rvr<rr_maybe_memory_mapr!r"r#rrtrrur6rjGzipFilerfBZ2File _BytesZipFileappendnamelistlenr}rlzmarr%reverser-)rr"r!r#rrrr]Zioargsr.handlesZis_pathrZ zip_namesrr1r+r+r, get_handle s<                      rcs^eZdZdZdddddfdd Zd d Zd d fd d ZfddZeddZ Z S)ra  Wrapper for standard library class ZipFile and allow the returned file-like handle to accept byte strings via `write` method. BytesIO provides attributes of file-like object and ZipFile.writestr writes bytes strings into a member of the archive. Nrr z str | None)filer" archive_namec sB|dd}||_d|_dtji}||tj||f|dS)Nr_rr#)rurmultiple_write_bufferzipfile ZIP_DEFLATEDupdatesuper__init__)r=rr"rrWZ kwargs_zip) __class__r+r,rs    z_BytesZipFile.__init__cCs2|jdkr"t|trtnt|_|j|dS)N)rr5rrrr^)r=datar+r+r,r^s z_BytesZipFile.writer3)r4c sP|jdks|jjrdS|jp$|jp$d}|jt||jWdQRXdS)Nr)rclosedrrrwritestrgetvalue)r=r)rr+r,r7s z_BytesZipFile.flushcs|tdS)N)r7rr:)r=)rr+r,r:'sz_BytesZipFile.closecCs |jdkS)N)fp)r=r+r+r,r+sz_BytesZipFile.closed)N) r&r'r(r)rr^r7r:propertyr __classcell__r+r+)rr,rs  rc@sbeZdZdZddddddd d Zdd d d ZddddZddddddZddddZdS) _MMapWrappera Wrapper for the Python's mmap class so that it can be properly read in by Python's csv.reader class. Parameters ---------- f : file object File object to be mapped onto memory. Must support the 'fileno' method or have an equivalent attribute utf-8rTr r r$)fr!rdecodecCsr||_||_t||d|_||_i|_x*dD]"}t||s@q0t|||j|<q0Wt j | dt j d|_ dS)N)r)seekablereadableZ writeabler)access) r!rcodecsgetincrementaldecoderdecoderr attributesrpgetattrrfileno ACCESS_READ)r=rr!rr attributer+r+r,r=s  z_MMapWrapper.__init__)namecs$jkrfddStjS)Ncs jS)N)rr+)rr=r+r,Rz*_MMapWrapper.__getattr__..)rrr)r=rr+)rr=r, __getattr__Ps z_MMapWrapper.__getattr__)r4cCs|S)Nr+)r=r+r+r,__iter__Usz_MMapWrapper.__iter__intz str | bytes)sizer4cCs:|j|}|jr6|dkp$t||k}|jj||dS|S)Nr)final)rrzrrr)r=rcontentrr+r+r,rzXs  z_MMapWrapper.readcCs,|j}|j|}|dkr"t|dS)Nr )rreadlinerr StopIterationlstrip)r=Znewbytesrr+r+r,__next__bs   z_MMapWrapper.__next__N)rrT)r) r&r'r(r)rrrrzrr+r+r+r,r0s  rrz'tuple[FileOrBuffer, bool, list[Buffer]])r.rr!r"rrr4cCsg}|t|dpt|tM}|s*|||fSt|trh|rTd|krTt||||dd}n t||}||y:ttjt||||}|| ||||}Wnt k rd}YnX|||fS)zTry to memory map file/buffer.rr_r)r!rrF) rpr5r r}rr rrr:r9 Exception)r.rr!r"rrrwrappedr+r+r,rss(         rc CsHd}t|}t|ts|Sytj|}Wnttfk rBYnX|S)zTest whether file exists.F)rSr5r rIrJexistsrMr<)rrr+r+r, file_existss r)r.r"r4cCs`d|ksd|krd|kStjtjtjtjf}tt||r>dStt f}t ||p^dt |d|kS)z+Whether the handle is opened in binary moderhr_Fr") r StreamWriter StreamReaderStreamReaderWritertempfileSpooledTemporaryFile issubclassrrrr5r)r.r"Z text_classesZbinary_classesr+r+r,rsr)F)r[Nr\N)NNFTNN)Ir) __future__rrfr collectionsrrArjiorrrrrrrIrtypingr r r r r urllib.parserrrDrrrrqrZpandas._typingrrrrrrZ pandas.compatrrZpandas.compat._optionalrZpandas.core.dtypes.commonrrsetrFdiscard dataclassrr-rHrLrOrSrVrZrrrrnrorZipFilerIteratorrrrrr+r+r+r,sd      0   0#> f9C-