o
    ŀg                     @   s   d Z ddlmZ ddlmZ ddlmZ ddlZddl	Z	ddl
Zddl
mZ ddlmZ ddlmZ e	jdZe	jje	jjgZe	jd	d
d Zdd Zedd ZdS )zd
Tests multithreading behaviour for reading and
parsing files for each parser defined in parsers.py
    )	ExitStack)BytesIO)
ThreadPoolN)	DataFrame)Versionpyarrow_xfailz0ignore:Passing a BlockManager:DeprecationWarningc                    s   | }|j dkrtd}t|jtdk r|tjjdd d d} fddt|D }t	 /fd	d
|D }
td}||j|}|d }	|D ]}
t|	|
 qOW d    d S 1 scw   Y  d S )Npyarrowz16.0z+# ValueError: Found non-unique column index)reasond   
   c                 3   s,    | ]}d  dd t D  V  qdS )
c                 S   s&   g | ]}|d d|d d|d qS )d, .0ir   r   \/var/www/html/myenv/lib/python3.10/site-packages/pandas/tests/io/parser/test_multi_thread.py
<listcomp>)   s   & zBtest_multi_thread_string_io_read_csv.<locals>.<genexpr>.<listcomp>N)joinrangeencode)r   _)max_row_ranger   r   	<genexpr>(   s
    
z7test_multi_thread_string_io_read_csv.<locals>.<genexpr>c                    s   g | ]	}  t|qS r   )enter_contextr   )r   b)stackr   r   r   /   s    z8test_multi_thread_string_io_read_csv.<locals>.<listcomp>   r   )enginepytestimportorskipr   __version__applymarkermarkxfailr   r   r   r   mapread_csvtmassert_frame_equal)all_parsersrequestparserpa	num_filesbytes_to_dffilespoolresultsfirst_resultresultr   )r   r   r   $test_multi_thread_string_io_read_csv   s*   


"r5   c                    s   fdd} fddt D }td}|||}W d   n1 s)w   Y  |d j}|dd D ]}	||	_q9t|}
|
S )	a  
    Generate a DataFrame via multi-thread.

    Parameters
    ----------
    parser : BaseParser
        The parser object to use for reading the data.
    path : str
        The location of the CSV file to read.
    num_rows : int
        The number of rows to read per task.
    num_tasks : int
        The number of tasks to use for reading this DataFrame.

    Returns
    -------
    df : DataFrame
    c                    sB   | \}}|s j dd|dgdS  j ddt|d |dgdS )aj  
        Create a reader for part of the CSV.

        Parameters
        ----------
        arg : tuple
            A tuple of the following:

            * start : int
                The starting row to start for parsing CSV
            * nrows : int
                The number of rows to read.

        Returns
        -------
        df : DataFrame
        r   date)	index_colheadernrowsparse_datesN   	   )r7   r8   skiprowsr9   r:   )r'   int)argstartr9   )r,   pathr   r   readerN   s   
z0_generate_multi_thread_dataframe.<locals>.readerc                    s    g | ]} |    fqS r   r   r   )num_rows	num_tasksr   r   r   p   s    z4_generate_multi_thread_dataframe.<locals>.<listcomp>)	processesNr   r;   )r   r   r&   columnspdconcat)r,   rA   rC   rD   rB   tasksr1   r2   r8   rfinal_dataframer   )rC   rD   r,   rA   r    _generate_multi_thread_dataframe:   s   "

rL   c                 C   s   d}d}| }d}t tjd|tjd|tjd|tjd|tjd|dg| dg| dg| tjd|d	d
tj|ddd
}t|}|	| t
||||}t|| W d    d S 1 sqw   Y  d S )N   0   z__thread_pool_reader__.csv   foobarbazz20000101 09:00:00s)periodsfreqint64)dtype)
ar   cr   erP   rQ   rR   r6   r>   )r   nprandomdefault_rngrG   
date_rangearanger(   ensure_cleanto_csvrL   r)   )r*   rD   rC   r,   	file_namedfrA   rK   r   r   r   )test_multi_thread_path_multipart_read_csv   s0   
"rd   )__doc__
contextlibr   ior   multiprocessing.poolr   numpyr[   r    pandasrG   r   pandas._testing_testingr(   pandas.util.versionr   r$   usefixturesxfail_pyarrow
single_cpuslow
pytestmarkfilterwarningsr5   rL   rd   r   r   r   r   <module>   s&    

F