§
    !P„j|:  ã                   ó0  — d Z ddlmZmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZ eZg Z	 ddlmZ n # e$ r e                     d¦  «         Y nw xY w	 ddlmZ n # e$ r e                     d¦  «         Y nw xY w	 ddlmZ n # e$ r e                     d	¦  «         Y nw xY w	 dd
lmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ ddl%Z% e%j&        e"¦  «         n # e$ r e                     d¦  «         Y nw xY wd„ Z'ddœd„Z(dddœd„Z)	 ddddœd„Z* G d„ de¦  «        Z+dS )zO
FileSystem abstraction to interact with various local and remote filesystems.
é    )Ú_is_path_likeÚ_stringify_path)ÚFileSelectorÚFileTypeÚFileInfoÚ
FileSystemÚLocalFileSystemÚSubTreeFileSystemÚ_MockFileSystemÚFileSystemHandlerÚPyFileSystemÚ_copy_filesÚ_copy_files_selector)ÚAzureFileSystemr   )ÚHadoopFileSystemr   )ÚGcsFileSystemr   )
ÚAwsDefaultS3RetryStrategyÚAwsStandardS3RetryStrategyÚS3FileSystemÚ
S3LogLevelÚS3RetryStrategyÚensure_s3_initializedÚfinalize_s3Úensure_s3_finalizedÚinitialize_s3Úresolve_s3_regionNr   c                 ó`   — | t           v rt          d| › d�¦  «        ‚t          d| › d�¦  «        ‚)Nz8The pyarrow installation is not built with support for 'ú'z&module 'pyarrow.fs' has no attribute ')Ú_not_importedÚImportErrorÚAttributeError)Únames    úO/root/disk_3tb/conda_envs/Huy-visual/lib/python3.11/site-packages/pyarrow/fs.pyÚ__getattr__r$   I   sX   € Ø�}ÐÐÝðØðð ð ñ
ô 
ð 	
õ
 Ø8°Ð8Ð8Ð8ñô ð ó    F©Úuse_mmapc          	      óÌ  — t          | t          ¦  «        r| S t          | t          ¦  «        rª|rt          d¦  «        ‚t          j        | ¦  «        \  }}|                     |¦  «        }|ri|                     |g¦  «        d         }|j        t          j	        k    r(t          d|j        j
        › d|j        › d| › d�¦  «        ‚t          ||¦  «        }|S 	 dd l}t          | |j        ¦  «        rDt          | ¦  «        j        dk    rt!          |¬¦  «        S t#          t%          | ¦  «        ¦  «        S n# t&          $ r Y nw xY wt)          d	t          | ¦  «        › d
�¦  «        ‚)NzXSpecifying to use memory mapping not supported for filesystem specified as an URI stringr   zWThe path component of the filesystem URI must point to a directory but it has a type: `z`. The path component is `z#` and the given filesystem URI is `ú`r	   r&   zUnrecognized filesystem: zP. `filesystem` argument must be a FileSystem instance or a valid file system URI)Ú
isinstancer   ÚstrÚ
ValueErrorÚfrom_uriÚnormalize_pathÚget_file_infoÚtyper   Ú	Directoryr"   Úpathr
   ÚfsspecÚAbstractFileSystemÚ__name__r	   r   ÚFSSpecHandlerr    Ú	TypeError)Ú
filesystemr'   Úfsr2   ÚprefixÚprefix_infor3   s          r#   Ú_ensure_filesystemr<   U   sÍ  € Ý�*�jÑ)Ô)ð )
ØÐÝ	�J¥Ñ	$Ô	$ð '
ð ð 	Ýð8ñô ð õ Ô& zÑ2Ô2‰ˆˆDØ×"Ò" 4Ñ(Ô(ˆØð 
	/à×*Ò*¨F¨8Ñ4Ô4°QÔ7ˆKØÔ¥8Ô#5Ò5Ð5Ý ð)Ø5@Ô5EÔ5Jð)ð )à%0Ô%5ð)ð )ð &ð)ð )ð )ñô ð õ # 6¨2Ñ.Ô.ˆBØˆ	ð		?ØˆMˆMˆMõ ˜* fÔ&?Ñ@Ô@ð ?Ý˜
Ñ#Ô#Ô,Ð0AÒAÐAå*°HÐ=Ñ=Ô=Ð=Ý#¥M°*Ñ$=Ô$=Ñ>Ô>Ð>ð	?øõ ð 	ð 	ð 	ØˆDð	øøøõ ðB­¨ZÑ(8Ô(8ð Bð Bð Bñ
ô 
ð 	
s   ÃD6 Ä6
EÅE)Ú
memory_mapc                ó¼  — t          | ¦  «        s|�t          d¦  «        ‚|| fS |�st          ||¬¦  «        }t          |t          ¦  «        rt          | ¦  «        } n$t          | t          ¦  «        st          d¦  «        ‚|                     | ¦  «        } || fS t          | ¦  «        } t	          |¬¦  «        }	 | 	                    | ¦  «        }|j
        t          j        k    }n# t          $ r d}d}Y nw xY w|sJ	 t          j        | ¦  «        \  }} nF# t          $ r$}t          |¦  «        }d|v sd|v rn|‚Y d}~nd}~ww xY w|                     | ¦  «        } || fS )zŒ
    Return filesystem/path from path which could be an URI or a plain
    filesystem path or a combination of fsspec protocol and URI.
    Nzg'filesystem' passed but the specified path is file-like, so there is nothing to open with 'filesystem'.r&   zPExpected string path; path-like objects are only allowed with a local filesystemFzempty schemezCannot parse URI)r   r,   r<   r*   r	   r   r+   r7   r.   r/   r0   r   ÚNotFoundr   r-   )r2   r8   r=   Ú	file_infoÚexists_locallyÚeÚmsgs          r#   Ú_resolve_filesystem_and_pathrD   ‚   sÒ  € õ
 ˜ÑÔð  ØÐ!Ýð?ñô ð ð ˜4ÐÐàÐÝ'¨
¸ZÐHÑHÔHˆ
Ý�j¥/Ñ2Ô2ð 	Ý" 4Ñ(Ô(ˆDˆDÝ˜D¥#Ñ&Ô&ð 	Ýð*ñô ð ð ×(Ò(¨Ñ.Ô.ˆØ˜4ÐÐå˜4Ñ Ô €Dõ
 !¨*Ð5Ñ5Ô5€Jð?Ø×,Ò,¨TÑ2Ô2ˆ	ð
 $œ.­HÔ,=Ò=ˆˆøõ	 ð ð ð Øˆ	Øˆˆˆðøøøð ð /ð
	Ý)Ô2°4Ñ8Ô8ÑˆJ˜˜øÝð 	ð 	ð 	Ý�a‘&”&ˆCØ Ð$Ð$Ð(:¸cÐ(AÐ(Að à�ð ����øøøøð	øøøð ×(Ò(¨Ñ.Ô.ˆà�tÐÐs*   Â:C% Ã%C6Ã5C6Ã<D Ä
EÄD=Ä=Ei   T)Ú
chunk_sizeÚuse_threadsc                ó  — t          | |¦  «        \  }}t          ||¦  «        \  }}	|                     |¦  «        }
|
j        t          j        k    r't          |d¬¦  «        }t          ||||	||¦  «         dS t          ||||	||¦  «         dS )a–  
    Copy files between FileSystems.

    This functions allows you to recursively copy directories of files from
    one file system to another, such as from S3 to your local machine.

    Parameters
    ----------
    source : string
        Source file path or URI to a single file or directory.
        If a directory, files will be copied recursively from this path.
    destination : string
        Destination file path or URI. If `source` is a file, `destination`
        is also interpreted as the destination file (not directory).
        Directories will be created as necessary.
    source_filesystem : FileSystem, optional
        Source filesystem, needs to be specified if `source` is not a URI,
        otherwise inferred.
    destination_filesystem : FileSystem, optional
        Destination filesystem, needs to be specified if `destination` is not
        a URI, otherwise inferred.
    chunk_size : int, default 1MB
        The maximum size of block to read before flushing to the
        destination file. A larger chunk_size will use more memory while
        copying but may help accommodate high latency FileSystems.
    use_threads : bool, default True
        Whether to use multiple threads to accelerate copying.

    Examples
    --------
    Inspect an S3 bucket's files:

    >>> s3, path = fs.FileSystem.from_uri(
    ...            "s3://registry.opendata.aws/roda/ndjson/")
    >>> selector = fs.FileSelector(path)
    >>> s3.get_file_info(selector)
    [<FileInfo for 'registry.opendata.aws/roda/ndjson/index.ndjson':...]

    Copy one file from S3 bucket to a local directory:

    >>> fs.copy_files("s3://registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               f"file:///{local_path}/index_copy.ndjson")

    >>> fs.LocalFileSystem().get_file_info(str(local_path)+
    ...                                    '/index_copy.ndjson')
    <FileInfo for '.../index_copy.ndjson': type=FileType.File, size=...>

    Copy file using a FileSystem object:

    >>> fs.copy_files("registry.opendata.aws/roda/ndjson/index.ndjson",
    ...               f"file:///{local_path}/index_copy.ndjson",
    ...               source_filesystem=fs.S3FileSystem())
    T©Ú	recursiveN)rD   r/   r0   r   r1   r   r   r   )ÚsourceÚdestinationÚsource_filesystemÚdestination_filesystemrE   rF   Ú	source_fsÚsource_pathÚdestination_fsÚdestination_pathr@   Ú
source_sels               r#   Ú
copy_filesrS   ¾   sÌ   € õp :ØÐ!ñô Ñ€Iˆ{õ (DØÐ+ñ(ô (Ñ$€NÐ$ð ×'Ò'¨Ñ4Ô4€IØ„~�Ô+Ò+Ð+Ý! +¸Ð>Ñ>Ô>ˆ
Ý˜Y¨
Ø+Ð-=Ø'¨ñ	6ô 	6ð 	6ð 	6ð 	6õ 	�I˜{Ø"Ð$4Ø ñ	-ô 	-ð 	-ð 	-ð 	-r%   c                   óš   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Ze	d„ ¦   «         Z
d„ Zd	„ Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ ZdS )r6   a  
    Handler for fsspec-based Python filesystems.

    https://filesystem-spec.readthedocs.io/en/latest/index.html

    Parameters
    ----------
    fs : FSSpec-compliant filesystem instance

    Examples
    --------
    >>> PyFileSystem(FSSpecHandler(fsspec_fs)) # doctest: +SKIP
    c                 ó   — || _         d S ©N)r9   )Úselfr9   s     r#   Ú__init__zFSSpecHandler.__init__  s   € ØˆŒˆˆr%   c                 óZ   — t          |t          ¦  «        r| j        |j        k    S t          S rV   ©r*   r6   r9   ÚNotImplemented©rW   Úothers     r#   Ú__eq__zFSSpecHandler.__eq__  ó)   € Ý�e�]Ñ+Ô+ð 	'Ø”7˜eœhÒ&Ð&ÝÐr%   c                 óZ   — t          |t          ¦  «        r| j        |j        k    S t          S rV   rZ   r\   s     r#   Ú__ne__zFSSpecHandler.__ne__   r_   r%   c                 ó^   — | j         j        }t          |t          ¦  «        r|d         }d|› �S )Nr   zfsspec+)r9   Úprotocolr*   Úlist)rW   rc   s     r#   Úget_type_namezFSSpecHandler.get_type_name%  s5   € Ø”7Ô#ˆÝ�h¥Ñ%Ô%ð 	#Ø ”{ˆHØ#˜Ð#Ð#Ð#r%   c                 ó   — |S rV   © ©rW   r2   s     r#   r.   zFSSpecHandler.normalize_path+  s   € Øˆr%   c           	      óà   — |d         }|d         dk    rt           j        }n'|d         dk    rt           j        }d }nt           j        }t	          | |||                     dd ¦  «        ¬¦  «        S )NÚsizer0   ÚfileÚ	directoryÚmtime)rj   rm   )r   ÚFiler1   ÚUnknownr   Úget)r2   Úinforj   Úftypes       r#   Ú_create_file_infozFSSpecHandler._create_file_info.  sn   € à�FŒ|ˆØ�Œ<˜6Ò!Ð!Ý”MˆEˆEØ�&Œ\˜[Ò(Ð(ÝÔ&ˆEàˆDˆDåÔ$ˆEÝ˜˜e¨$°d·h²h¸wÈÑ6MÔ6MÐNÑNÔNÐNr%   c                 ó  — g }|D ]ƒ}	 | j                              |¦  «        }|                     |                      ||¦  «        ¦  «         ŒG# t          $ r0 |                     t          |t          j        ¦  «        ¦  «         Y Œ€w xY w|S rV   )r9   rq   Úappendrs   ÚFileNotFoundErrorr   r   r?   )rW   ÚpathsÚinfosr2   rq   s        r#   r/   zFSSpecHandler.get_file_info;  s¥   € ØˆØð 	Að 	AˆDðAØ”w—|’| DÑ)Ô)�ð —’˜T×3Ò3°D¸$Ñ?Ô?Ñ@Ô@Ð@Ð@øõ %ð @ð @ð @Ø—’�X d­HÔ,=Ñ>Ô>Ñ?Ô?Ð?Ð?Ð?ð@øøøð ˆs   ˆAÁ7BÂBc                 ó6  — | j                              |j        ¦  «        sP| j                              |j        ¦  «        rt	          |j        ¦  «        ‚|j        rg S t          |j        ¦  «        ‚|j        rd }nd}g }| j                              |j        |dd¬¦  «        }| 	                    ¦   «         D ]c\  }}| 
                    d¦  «        }|j         
                    d¦  «        }||k    r)|                     |                      ||¦  «        ¦  «         Œd|S )Né   T)ÚmaxdepthÚwithdirsÚdetailú/)r9   ÚisdirÚbase_dirÚexistsÚNotADirectoryErrorÚallow_not_foundrv   rI   ÚfindÚitemsÚstripru   rs   )	rW   Úselectorr{   rx   Úselected_filesr2   rq   Ú_pathr€   s	            r#   Úget_file_info_selectorz$FSSpecHandler.get_file_info_selectorF  s%  € ØŒw�}Š}˜XÔ.Ñ/Ô/ð 	?ØŒw�~Š~˜hÔ/Ñ0Ô0ð ?Ý(¨Ô):Ñ;Ô;Ð;àÔ+ð ?Ø�Iå+¨HÔ,=Ñ>Ô>Ð>àÔð 	ØˆHˆHàˆHàˆØœŸšØÔ¨¸4Èð &ñ 
ô 
ˆð )×.Ò.Ñ0Ô0ð 	Að 	A‰JˆD�$Ø—J’J˜s‘O”OˆEØÔ(×.Ò.¨sÑ3Ô3ˆHð ˜Ò Ð Ø—’˜T×3Ò3°D¸$Ñ?Ô?Ñ@Ô@Ð@øàˆr%   c                 ób   — 	 | j                              ||¬¦  «         d S # t          $ r Y d S w xY w)N)Úcreate_parents)r9   ÚmkdirÚFileExistsError)rW   r2   rI   s      r#   Ú
create_dirzFSSpecHandler.create_dirc  sG   € ð	ØŒG�MŠM˜$¨yˆMÑ9Ô9Ð9Ð9Ð9øÝð 	ð 	ð 	ØˆDˆDð	øøøs   ‚   
.­.c                 ó>   — | j                              |d¬¦  «         d S ©NTrH   )r9   Úrmrh   s     r#   Ú
delete_dirzFSSpecHandler.delete_dirj  s    € ØŒ�
Š
�4 4ˆ
Ñ(Ô(Ð(Ð(Ð(r%   c                 óJ  — 	 | j                              |d¬¦  «        }n# t          $ r |rY d S ‚ w xY w|D ]m}| j                              |¦  «        r| j                              |d¬¦  «         Œ9| j                              |¦  «        r| j                              |¦  «         Œnd S )NF)r}   TrH   )r9   Úlistdirrv   r   r’   Úisfile)rW   r2   Úmissing_dir_okÚsubpathsÚsubpaths        r#   Ú_delete_dir_contentsz"FSSpecHandler._delete_dir_contentsm  sÄ   € ð	Ø”w—’ t°E�Ñ:Ô:ˆHˆHøÝ ð 	ð 	ð 	Øð Ø��Øð	øøøð  ð 	$ð 	$ˆGØŒw�}Š}˜WÑ%Ô%ð $Ø”—
’
˜7¨d�
Ñ3Ô3Ð3Ð3Ø”—’ Ñ(Ô(ð $Ø”—
’
˜7Ñ#Ô#Ð#øð		$ð 	$s   ‚ Ÿ0®0c                 ó†   — |                      d¦  «        dk    rt          d|d¦  «        ‚|                      ||¦  «         d S )Nr~   Ú z$delete_dir_contents called on path 'r   )r†   r,   rš   )rW   r2   r—   s      r#   Údelete_dir_contentsz!FSSpecHandler.delete_dir_contentsz  sO   € Ø�:Š:�c‰?Œ?˜bÒ Ð ÝØ6¸¸cñCô Cð Cà×!Ò! $¨Ñ7Ô7Ð7Ð7Ð7r%   c                 ó2   — |                       dd¦  «         d S )Nr~   F)rš   )rW   s    r#   Údelete_root_dir_contentsz&FSSpecHandler.delete_root_dir_contents€  s   € Ø×!Ò! # uÑ-Ô-Ð-Ð-Ð-r%   c                 óŒ   — | j                              |¦  «        st          |¦  «        ‚| j                              |¦  «         d S rV   )r9   r�   rv   r’   rh   s     r#   Údelete_filezFSSpecHandler.delete_fileƒ  sB   € ð Œw�~Š~˜dÑ#Ô#ð 	*Ý# DÑ)Ô)Ð)ØŒ�
Š
�4ÑÔÐÐÐr%   c                 ó@   — | j                              ||d¬¦  «         d S r‘   )r9   Úmv©rW   ÚsrcÚdests      r#   ÚmovezFSSpecHandler.moveŠ  s"   € ØŒ�
Š
�3˜¨ˆ
Ñ-Ô-Ð-Ð-Ð-r%   c                 ó<   — | j                              ||¦  «         d S rV   )r9   Úcopyr¤   s      r#   Ú	copy_filezFSSpecHandler.copy_file�  s    € ð 	Œ�Š�S˜$ÑÔÐÐÐr%   c                 ó®   — ddl m} | j                             |¦  «        st	          |¦  «        ‚ || j                             |d¬¦  «        d¬¦  «        S ©Nr   ©Ú
PythonFileÚrb©ÚmodeÚr©Úpyarrowr®   r9   r–   rv   Úopen©rW   r2   r®   s      r#   Úopen_input_streamzFSSpecHandler.open_input_stream”  ó_   € Ø&Ð&Ð&Ð&Ð&Ð&àŒw�~Š~˜dÑ#Ô#ð 	*Ý# DÑ)Ô)Ð)àˆz˜$œ'Ÿ,š, t°$˜,Ñ7Ô7¸cÐBÑBÔBÐBr%   c                 ó®   — ddl m} | j                             |¦  «        st	          |¦  «        ‚ || j                             |d¬¦  «        d¬¦  «        S r¬   r³   r¶   s      r#   Úopen_input_filezFSSpecHandler.open_input_fileœ  r¸   r%   c                 ó\   — ddl m}  || j                             |d¬¦  «        d¬¦  «        S )Nr   r­   Úwbr°   Úw©r´   r®   r9   rµ   ©rW   r2   Úmetadatar®   s       r#   Úopen_output_streamz FSSpecHandler.open_output_stream¤  ó;   € Ø&Ð&Ð&Ð&Ð&Ð&àˆz˜$œ'Ÿ,š, t°$˜,Ñ7Ô7¸cÐBÑBÔBÐBr%   c                 ó\   — ddl m}  || j                             |d¬¦  «        d¬¦  «        S )Nr   r­   Úabr°   r½   r¾   r¿   s       r#   Úopen_append_streamz FSSpecHandler.open_append_stream©  rÂ   r%   N)r5   Ú
__module__Ú__qualname__Ú__doc__rX   r^   ra   re   r.   Ústaticmethodrs   r/   rŠ   r�   r“   rš   r�   rŸ   r¡   r§   rª   r·   rº   rÁ   rÅ   rg   r%   r#   r6   r6   	  sg  € € € € € ðð ðð ð ðð ð ð
ð ð ð
$ð $ð $ðð ð ð ð
Oð 
Oñ „\ð
Oð	ð 	ð 	ðð ð ð:ð ð ð)ð )ð )ð$ð $ð $ð8ð 8ð 8ð.ð .ð .ðð ð ð.ð .ð .ð ð  ð  ðCð Cð CðCð Cð CðCð Cð Cð
Cð Cð Cð Cð Cr%   r6   rV   )NN),rÈ   Úpyarrow.utilr   r   Úpyarrow._fsr   r   r   r   r	   r
   r   r   r   r   r   Ú	FileStatsr   Úpyarrow._azurefsr   r    ru   Úpyarrow._hdfsr   Úpyarrow._gcsfsr   Úpyarrow._s3fsr   r   r   r   r   r   r   r   r   r   ÚatexitÚregisterr$   r<   rD   rS   r6   rg   r%   r#   ú<module>rÓ      s[  ðð$ð ð 8Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7Ð 7ðð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð ð €	à€ð,Ø0Ð0Ð0Ð0Ð0Ð0Ð0øØð ,ð ,ð ,Ø×ÒÐ*Ñ+Ô+Ð+Ð+Ð+ð,øøøð-Ø.Ð.Ð.Ð.Ð.Ð.Ð.øØð -ð -ð -Ø×ÒÐ+Ñ,Ô,Ð,Ð,Ð,ð-øøøð*Ø,Ð,Ð,Ð,Ð,Ð,Ð,øØð *ð *ð *Ø×Ò˜Ñ)Ô)Ð)Ð)Ð)ð*øøøð)ðLð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð Lð €M€M€MØ€F„OÐ'Ñ(Ô(Ð(Ð(øð ð )ð )ð )Ø×Ò˜Ñ(Ô(Ð(Ð(Ð(ð)øøøð	ð 	ð 	ð 05ð *
ð *
ð *
ð *
ð *
ðZ9Àeð 9ð 9ð 9ð 9ð 9ðz ?CðH-à&°DðH-ð H-ð H-ð H-ð H-ðVcCð cCð cCð cCð cCÐ%ñ cCô cCð cCð cCð cCsE   ª1 ±AÁAÁA ÁA6Á5A6Á:B ÂBÂBÂ"C ÃC,Ã+C,