o
    ŀgzV                     @   s  d Z ddlmZ ddlZddlZddlmZ ddlm	Z	m
Z
mZ ddlmZ ejdZejdZejdZd	d
 Zdd Zejdddgddgddgdgdgdgdggejdddgdd Zdd Zejdddggdd Zdd Zdd Zejd dge	ejgd!gd"gd#e
dgd$d%d&fdd'ge	ejgd"gd(ejd)gd$d*gd+d&fd$d*ge	ejgd"gd(ejd)gd$d*gd+d&fgd,d- Zejd.i e	d$d/ejd0d1ejd2gg d3d4d5d6ejd7ejd8gd9fg g d:d;d<e	g d=g d3g d>d9fd$gd;d<e	ejd/d?d0d1d@d2gg d3g d>d9fdg g d:ie	d$d/ejd0d1ejd2gg d3d4d5d6ejd7ejd8gd9fgdAdB Z dCdD Z!dEdF Z"dGdH Z#ejdIdJdKgdLdM Z$eejdNdOd!dPgejejgdQdRggfd;dSdPgd@dTgdUdRggfgdVdW Z%edXdY Z&ejdZd!ejd[gd[ejggfd'd!d\d]d[gejejggfgd^d_ Z'd`da Z(dbdc Z)ejdde*dedf e*dg ddegie	e*dee*dggfe*dedh di i e	e*ded!gd?d'ggfe*dedj ddegie	ejd!gfgdkdl Z+dmdn Z,ejdod;d?dpgfdOejdqgfgdrds Z-dtdu Z.eejdvdOd;gdwdx Z/eejdydzd{d|d}d~dgfd~d$difgdd Z0edd Z1dd Z2edd Z3dd Z4dd Z5dS )zg
Tests that NA values are properly handled during
parsing for all of the parsers defined in parsers.py
    )StringION)STR_NA_VALUES)	DataFrameIndex
MultiIndexz=ignore:Passing a BlockManager to DataFrame:DeprecationWarningpyarrow_xfailpyarrow_skipc                 C   sn   | }d}| t|}tg ddtjdgtjddggg dd}|jd	kr/d |jd
< d |jd< t|| d S )NzA,B,C
a,b,c
d,,f
,g,h
abcdfghABCcolumnspyarrow)   r   )   r   )	read_csvr   r   npnanengineloctmassert_frame_equalall_parsersparserdataresultexpected r'   Y/var/www/html/myenv/lib/python3.10/site-packages/pandas/tests/io/parser/test_na_values.pytest_string_nas   s   


r)   c                 C   st   | }d}t ddgtjdgtjtjggddgd}|jdkr+d |jd	d
gdf< d |jd< |t|}t|| d S )NzA,B
foo,bar
NA,baz
NaN,nan
foobarbazr   r   r   r   r   r   r   r   )	r   r   r   r   r   r   r   r   r    )r"   r#   r$   r&   r%   r'   r'   r(   test_detect_string_na-   s    

r.   	na_valuesz-999.0z-999ig     8r$   zA,B
-999,1.2
2,-999
3,4.5
z"A,B
-999,1.200
2,-999.000
3,4.500
c           	      C   s   | }t tjdgdtjgddggddgd}|jdkrHtd	d
 |D sHd}tjt|d |jt	||d W d    d S 1 sAw   Y  d S |jdkr]d|v r]tj
jdd}|| |jt	||d}t|| d S )Ng333333?       @g      @g      @r   r   r   r   c                 s   s    | ]}t |tV  qd S )N)
isinstancestr).0xr'   r'   r(   	<genexpr>_   s    z,test_non_string_na_values.<locals>.<genexpr>9The 'pyarrow' engine requires all na_values to be stringsmatchr/   z-999.000z4pyarrow engined does not recognize equivalent floatsreason)r   r   r   r   allpytestraises	TypeErrorr   r   markxfailapplymarkerr   r    )	r"   r$   r/   requestr#   r&   msgr@   r%   r'   r'   r(   test_non_string_na_values>   s"   &

rE   c                    s   h d}|t ks
J | }t|fdd td fddt|D }ttjttd}|j	|d d}t
|| d S )	N>   #NAN/An/a#N/A-NaN-nan<NA>1.#IND-1.#IND1.#QNAN#N/A N/A-1.#QNAN NANaNr   NULLNonenullc                    sf   | dkrd}n| dkrd dg|  }| | }|  d k r1d dg |  d  }| | }|S )Nr   rR   ,r   )join)ivbufjoined)nvr'   r(   r      s   z!test_default_na_values.<locals>.f
c                    s   g | ]	\}} ||qS r'   r'   )r3   rZ   r[   )r   r'   r(   
<listcomp>   s    z*test_default_na_values.<locals>.<listcomp>)r   index)header)r   lenr   rY   	enumerater   r   r   ranger   r   r    )r"   
_NA_VALUESr#   r$   r&   r%   r'   )r   r^   r(   test_default_na_valuesq   s    rg   r,   c                 C   s   | }d}t dtjdgtjdtjgddtjggg dd}|jd	krHd
}tjt|d |jt||dgd W d    d S 1 sAw   Y  d S |jt||dgd}t	
|| d S )Nz3A,B,C
ignore,this,row
1,NA,3
-1.#IND,5,baz
7,8,NaN
      ?            r   r   r   z@skiprows argument must be an integer when using engine='pyarrow'r7   r   )r/   skiprowsr   r   r   r   r=   r>   
ValueErrorr   r   r   r    )r"   r/   r#   r$   r&   rD   r%   r'   r'   r(   test_custom_na_values   s   (

rp   c                 C   s|   d}| }| t|}ttjdtjdgtdtjddtjgtdg dd}|jdkr6d |jd< d |jd	< t	
|| d S )
Nz1A,B,C
True,False,True
NA,True,False
False,NA,TrueTFdtype)TFTr   r   )r   r   r-   )r   r   r   r   arrayr   objectr   r   r   r    r"   r$   r#   r%   r&   r'   r'   r(   test_bool_na_values   s   


rv   c                 C   s   d}| }|j dkr3d}tjt|d |jt|dgdgdd W d    d S 1 s,w   Y  d S |jt|dgdgdd}ttjdtjdgtjdtjdgtjdtjdgd	}t	
|| d S )
Nz3A,B,C
foo,bar,NA
bar,foo,foo
foo,bar,NA
bar,foo,foor   z;pyarrow engine doesn't support passing a dict for na_valuesr7   r*   r+   )r   r   r9   r   r   r=   r>   ro   r   r   r   r   r   r   r    r"   r$   r#   rD   dfr&   r'   r'   r(   test_na_value_dict   s$   

rz   zindex_col,expectedr   rj   )r   r   r   r
   namera   r   )r   r   )r   r   r   )namesc                 C   s.   d}| }|j t|t |d}t|| d S )Nza,b,c,d
0,NA,1,5
)r/   	index_col)r   r   setr   r    )r"   r   r&   r$   r#   r%   r'   r'   r(   test_na_value_dict_multi_index   s   r   zkwargs,expectedr   r   er   r   r   ri      rj      rk   onetwothreefivesevenr   )r   r   Fr/   keep_default_nar
   r   rR   r   r   r   r   )r   r   r   r   r   rR   r   rR   r   c           	      C   s   d}| }|j dkrEd|v r;t|d tr;d}tjt|d |jt|fi | W d    d S 1 s4w   Y  d S tj	 }|
| |jt|fi |}t|| d S )NzAA,B,C
a,1,one
b,2,two
,3,three
d,4,nan
e,5,five
nan,6,
g,7,seven
r   r/   ?The pyarrow engine doesn't support passing a dict for na_valuesr7   )r   r1   dictr=   r>   ro   r   r   r@   rA   rB   r   r    )	r"   kwargsr&   rC   r$   r#   rD   r@   r%   r'   r'   r(   test_na_values_keep_default  s   .




r   c                 C   sF   d}| }|j t|dd}tg dg dg dd}t|| d S )NzAA,B,C
a,1,None
b,2,two
,3,None
d,4,nan
e,5,five
nan,6,
g,7,seven
F)r   r   r   )rV   r   rV   r   r   rR   r   r   r   r   r   r   r    ru   r'   r'   r(   !test_no_na_values_no_keep_defaultK  s   
r   c                 C   s   d}| }|j dkr2d}tjt|d |jt|ddgidd W d    d S 1 s+w   Y  d S |jt|ddgidd}td	gtjgd
}t	
|| d S )Nza,b
,2r   r   r7   r   2Fr   rR   r
   r   rw   )r"   r$   r#   rD   r%   r&   r'   r'   r(   &test_no_keep_default_na_dict_na_valuese  s"   

r   c                 C   s   d}| }|j dkr1d}tjt|d |jt|ddidd W d    d S 1 s*w   Y  d S |jt|ddidd}td	gtjgd
}t	
|| d S )Nza,b
1,2r   r   r7   r   r   Fr   r   r   rw   rx   r'   r'   r(   -test_no_keep_default_na_dict_na_scalar_valuesy  s   

r   col_zero_na_valuesi 113125c              
   C   s   d}| }t tjdgtjdgdtjgddgddgd	d
gtjdgd}|jdkrSd}tjt|d |jt|d dd
dd|dd W d    d S 1 sLw   Y  d S |jt|d dd
dd|dd}t	
|| d S )Nz_113125,"blah","/blaha",kjsdkj,412.166,225.874,214.008
729639,"qwer","",asdfkj,466.681,,252.373
g    ND&Aqwerz/blahakjsdkjasdfkjg-y@g7A`*}@z225.874rR   g-o@)r   r   r   ri   r   rj   r   r   r   r7   Fz214.008blah)r   r   r   r   )rb   r   r/   rn   )r"   r   r$   r#   r&   rD   r%   r'   r'   r(   1test_no_keep_default_na_dict_na_values_diff_reprs  s@   

r   zna_filter,row_dataTr   ri   r   1r   3c                 C   s>   d}| }|j t|dg|d}t|ddgd}t|| d S )NzA,B
1,A
nan,B
3,C
r   )r/   	na_filterr   r   r   )r"   r   row_datar$   r#   r%   r&   r'   r'   r(   !test_na_values_na_filter_override  s
   	r   c              
   C   sf   | }d}| t|}tdddddtjtjtjgdddd	d
tjtjtjggg dd}t|| d S )NzlDate,Currency,Symbol,Type,Units,UnitPrice,Cost,Tax
2012-03-14,USD,AAPL,BUY,1000
2012-05-12,USD,SBUX,SELL,500z
2012-03-14USDAAPLBUYi  z
2012-05-12SBUXSELLi  )DateCurrencySymbolTypeUnits	UnitPriceCostTaxr   )r   r   r   r   r   r   r    r!   r'   r'   r(   test_na_trailing_columns  s   r   zna_values,row_datar0   r   rh   c           
      C   s  | }ddg}d}|j dkrDt|trDt|trt}d}nt}d}tj||d |jt|||d W d    d S 1 s=w   Y  d S |j dkrod}tjt|d |jt|||d W d    d S 1 shw   Y  d S |jt|||d}t	||d	}	t
||	 d S )
Nr
   r   1,2
2,1r   r   r6   r7   r~   r/   r   )r   r1   r   ro   r?   r=   r>   r   r   r   r   r    )
r"   r/   r   r#   r~   r$   errrD   r%   r&   r'   r'   r(   test_na_values_scalar  s4   	



r   c           	      C   s   | }ddd}|  }ddg}d}tddgtjtjgg|d	}|jd
krJd}tjt|d |jt	|||d W d    d S 1 sCw   Y  d S |jt	|||d}t
|| t
|| d S )Nr   r   r   r
   r   r   rh   r0   r   r   r   r7   r   )copyr   r   r   r   r=   r>   ro   r   r   r   r    assert_dict_equal)	r"   r#   r/   na_values_copyr~   r$   r&   rD   r%   r'   r'   r(   test_na_values_dict_aliasing  s"   


r   c                 C   s   d}| }ddi}|j dkr2d}tjt|d |jt||d W d    d S 1 s+w   Y  d S |jt||d}tdtjd	gi}t	
|| d S )
Nza
foo
1r   r*   r   r   r7   r9   r
   r   rw   )r"   r$   r#   r/   rD   r%   r&   r'   r'   r(   test_na_values_dict_col_index  s   

r   zdata,kwargs,expectedl            r_   l           z,1z
,2z
1c           	      C   s   | }|j dkr4d|v r4d}tjt|d |jt|fdd i| W d    d S 1 s-w   Y  d S |j dkrEtjjdd}|| |jt|fdd i|}t	
|| d S )Nr   r/   r6   r7   rb   z!Returns float64 instead of objectr:   )r   r=   r>   r?   r   r   r@   rA   rB   r   r    )	r"   r$   r   r&   rC   r#   rD   r@   r%   r'   r'   r(   test_na_values_uint64.  s   


r   c                 C   sH   d}| }t ddgitdgddd}|jt|dd	d
}t|| d S )Nza,1
b,2r   r   r   r
   r{   r}   r   F)r   r   )r   r   r   r   r   r    )r"   r$   r#   r&   r%   r'   r'   r(   *test_empty_na_values_no_default_with_indexK  s
   r   zna_filter,index_data5g      @c           	      C   sz   | }d}|j dkr|du rtjjdd}|| tddgdd	gd
t|ddd}|jt|dg|d}t	
|| d S )Na,b,c
1,,3
4,5,6r   Fzmismatched index resultr:   r   r   ri   r   )r
   r   r   r{   r}   )r   r   )r   r=   r@   rA   rB   r   r   r   r   r   r    )	r"   r   
index_datarC   r#   r$   r@   r&   r%   r'   r'   r(   test_no_na_filter_on_indexU  s   
"r   c                 C   s\   | }d}|j t|dgddgd}tdtjgdtjgdtd	d
gddd}t|| d S )Nzidx,col1,col2
1,3,4
2,inf,-infr   infz-inf)r   r/   ri   r   )col1col2r   r   idxr{   r}   )r   r   r   r   r   r   r   r    )r"   r#   r$   outr&   r'   r'   r(   !test_inf_na_values_with_int_indexh  s   "r   r   c                 C   sV   | }d}|r	t jnd}tddg|dgddgd}|jt||td	}t|| d S )
Nr   rR   r   4r   r   6r	   )r   rr   )r   r   r   r   r   r2   r   r    )r"   r   r#   r$   emptyr&   r%   r'   r'   r(   +test_na_values_with_dtype_str_and_na_filteru  s   r   zdata, na_values)zfalse,1
,1
trueN)zfalse,1
null,1
trueN)zfalse,1
nan,1
trueN)false,1
foo,1
truer*   r   r*   c                 C   sf   | }d ddg}tjt|d |jt|d ddgddi|d W d    d S 1 s,w   Y  d S )	N|z(Bool column has NA values in column [0a]zRcannot safely convert passed user dtype of bool for object dtyped data in column 0r7   r
   r   bool)rb   r~   rr   r/   )rY   r=   r>   ro   r   r   )r"   r$   r/   r#   rD   r'   r'   r(   !test_cast_NA_to_bool_raises_error  s   "r   c                 C   sb   | }d}|j t|d g dtttdd }tddgddgdd	gdd
dgd}t|| d S )NzDFile: small.csv,,
10010010233,0123,654
foo,,bar
01001000155,4530,898)r   r   col3)rb   r~   rr   100100102330100100015501234530654898r   ri   r}   )r   r   r2   dropnar   r   r    r!   r'   r'   r(   test_str_nan_dropped  s$   
	r   c                 C   s   | }d}|j dkr5d}tjt|d |jt|ttdddid W d    d S 1 s.w   Y  d S |jt|ttdddid}td	gdgt	j
gd
}t|| d S )NzA,B,B
X,Y,Z
1,2,infr   r   r7   r   r   Zr   )rb   r/   r   ))r   X)r   Yr   )r   r=   r>   ro   r   r   listre   r   r   r   r   r    )r"   r#   r$   rD   r%   r&   r'   r'   r(   test_nan_multi_index  s,   

r   c                 C   N   | }d}t jtdd |jt|dd W d    d S 1 s w   Y  d S )N0
NaN
True
False
z	NA valuesr7   r   rq   r=   r>   ro   r   r   r"   r#   r$   r'   r'   r(   test_bool_and_nan_to_bool  s
   "r   c                 C   r   )Nr   zconvert|NoneTyper7   intrq   r   r   r'   r'   r(   test_bool_and_nan_to_int  s
   "r   c                 C   s@   | }d}|j t|dd}tdtjddgi}t|| d S )Nr   floatrq   0rh   g        )r   r   r   	from_dictr   r   r   r    r!   r'   r'   r(   test_bool_and_nan_to_float  s
   r   )6__doc__ior   numpyr   r=   pandas._libs.parsersr   pandasr   r   r   pandas._testing_testingr   r@   filterwarnings
pytestmarkusefixturesxfail_pyarrowskip_pyarrowr)   r.   parametrizerE   rg   rp   rv   rz   r   from_tuplesr   r   r   r   r   r   r   r   r   r   r   r2   r   r   r   r   r   r   r   r   r   r   r   r'   r'   r'   r(   <module>   s:   0
$



-
'

( 





