FM / parser

/usr/local/lib64/python3.6/site-packages/pandas/tests/io/parser/

/usr/local/lib64/python3.6/site-packages/pandas/tests/io/parser

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
conftest.py	2798	0644	edit dl rm
test_comment.py	3815	0644	edit dl rm
test_common.py	62825	0644	edit dl rm
test_compression.py	4612	0644	edit dl rm
test_converters.py	3989	0644	edit dl rm
test_c_parser_only.py	19872	0644	edit dl rm
test_dialect.py	4090	0644	edit dl rm
test_dtypes.py	16817	0644	edit dl rm
test_encoding.py	5966	0644	edit dl rm
test_header.py	15921	0644	edit dl rm
test_index_col.py	6822	0644	edit dl rm
test_mangle_dupes.py	3868	0644	edit dl rm
test_multi_thread.py	3485	0644	edit dl rm
test_na_values.py	15082	0644	edit dl rm
test_network.py	9459	0644	edit dl rm
test_parse_dates.py	46157	0644	edit dl rm
test_python_parser_only.py	9485	0644	edit dl rm
test_quoting.py	5078	0644	edit dl rm
test_read_fwf.py	18789	0644	edit dl rm
test_skiprows.py	7010	0644	edit dl rm
test_textreader.py	10714	0644	edit dl rm
test_unsupported.py	4268	0644	edit dl rm
test_usecols.py	16544	0644	edit dl rm
__init__.py	0	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/pandas/tests/io/parser/test_comment.py (3815B)

"""
Tests that comments are properly handled during parsing
for all of the parsers defined in parsers.py
"""
from io import StringIO

import numpy as np
import pytest

from pandas import DataFrame
import pandas._testing as tm

@pytest.mark.parametrize("na_values", [None, ["NaN"]])
def test_comment(all_parsers, na_values):
    parser = all_parsers
    data = """A,B,C
1,2.,4.#hello world
5.,NaN,10.0
"""
    expected = DataFrame(
        [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
    )
    result = parser.read_csv(StringIO(data), comment="#", na_values=na_values)
    tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize(
    "read_kwargs", [dict(), dict(lineterminator="*"), dict(delim_whitespace=True)]
)
def test_line_comment(all_parsers, read_kwargs):
    parser = all_parsers
    data = """# empty
A,B,C
1,2.,4.#hello world
#ignore this line
5.,NaN,10.0
"""
    if read_kwargs.get("delim_whitespace"):
        data = data.replace(",", " ")
    elif read_kwargs.get("lineterminator"):
        if parser.engine != "c":
            pytest.skip("Custom terminator not supported with Python engine")

data = data.replace("\n", read_kwargs.get("lineterminator"))

read_kwargs["comment"] = "#"
    result = parser.read_csv(StringIO(data), **read_kwargs)

expected = DataFrame(
        [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
    )
    tm.assert_frame_equal(result, expected)

def test_comment_skiprows(all_parsers):
    parser = all_parsers
    data = """# empty
random line
# second empty line
1,2,3
A,B,C
1,2.,4.
5.,NaN,10.0
"""
    # This should ignore the first four lines (including comments).
    expected = DataFrame(
        [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
    )
    result = parser.read_csv(StringIO(data), comment="#", skiprows=4)
    tm.assert_frame_equal(result, expected)

def test_comment_header(all_parsers):
    parser = all_parsers
    data = """# empty
# second empty line
1,2,3
A,B,C
1,2.,4.
5.,NaN,10.0
"""
    # Header should begin at the second non-comment line.
    expected = DataFrame(
        [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
    )
    result = parser.read_csv(StringIO(data), comment="#", header=1)
    tm.assert_frame_equal(result, expected)

def test_comment_skiprows_header(all_parsers):
    parser = all_parsers
    data = """# empty
# second empty line
# third empty line
X,Y,Z
1,2,3
A,B,C
1,2.,4.
5.,NaN,10.0
"""
    # Skiprows should skip the first 4 lines (including comments),
    # while header should start from the second non-commented line,
    # starting with line 5.
    expected = DataFrame(
        [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"]
    )
    result = parser.read_csv(StringIO(data), comment="#", skiprows=4, header=1)
    tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize("comment_char", ["#", "~", "&", "^", "*", "@"])
def test_custom_comment_char(all_parsers, comment_char):
    parser = all_parsers
    data = "a,b,c\n1,2,3#ignore this!\n4,5,6#ignorethistoo"
    result = parser.read_csv(
        StringIO(data.replace("#", comment_char)), comment=comment_char
    )

expected = DataFrame([[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"])
    tm.assert_frame_equal(result, expected)

@pytest.mark.parametrize("header", ["infer", None])
def test_comment_first_line(all_parsers, header):
    # see gh-4623
    parser = all_parsers
    data = "# notes\na,b,c\n# more notes\n1,2,3"

if header is None:
        expected = DataFrame({0: ["a", "1"], 1: ["b", "2"], 2: ["c", "3"]})
    else:
        expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"])

result = parser.read_csv(StringIO(data), comment="#", header=header)
    tm.assert_frame_equal(result, expected)

""" Tests that comments are properly handled during parsing for all of the parsers defined in parsers.py """ from io import StringIO import numpy as np import pytest from pandas import DataFrame import pandas._testing as tm @pytest.mark.parametrize("na_values", [None, ["NaN"]]) def test_comment(all_parsers, na_values): parser = all_parsers data = """A,B,C 1,2.,4.#hello world 5.,NaN,10.0 """ expected = DataFrame( [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"] ) result = parser.read_csv(StringIO(data), comment="#", na_values=na_values) tm.assert_frame_equal(result, expected) @pytest.mark.parametrize( "read_kwargs", [dict(), dict(lineterminator="*"), dict(delim_whitespace=True)] ) def test_line_comment(all_parsers, read_kwargs): parser = all_parsers data = """# empty A,B,C 1,2.,4.#hello world #ignore this line 5.,NaN,10.0 """ if read_kwargs.get("delim_whitespace"): data = data.replace(",", " ") elif read_kwargs.get("lineterminator"): if parser.engine != "c": pytest.skip("Custom terminator not supported with Python engine") data = data.replace("\n", read_kwargs.get("lineterminator")) read_kwargs["comment"] = "#" result = parser.read_csv(StringIO(data), **read_kwargs) expected = DataFrame( [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"] ) tm.assert_frame_equal(result, expected) def test_comment_skiprows(all_parsers): parser = all_parsers data = """# empty random line # second empty line 1,2,3 A,B,C 1,2.,4. 5.,NaN,10.0 """ # This should ignore the first four lines (including comments). expected = DataFrame( [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"] ) result = parser.read_csv(StringIO(data), comment="#", skiprows=4) tm.assert_frame_equal(result, expected) def test_comment_header(all_parsers): parser = all_parsers data = """# empty # second empty line 1,2,3 A,B,C 1,2.,4. 5.,NaN,10.0 """ # Header should begin at the second non-comment line. expected = DataFrame( [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"] ) result = parser.read_csv(StringIO(data), comment="#", header=1) tm.assert_frame_equal(result, expected) def test_comment_skiprows_header(all_parsers): parser = all_parsers data = """# empty # second empty line # third empty line X,Y,Z 1,2,3 A,B,C 1,2.,4. 5.,NaN,10.0 """ # Skiprows should skip the first 4 lines (including comments), # while header should start from the second non-commented line, # starting with line 5. expected = DataFrame( [[1.0, 2.0, 4.0], [5.0, np.nan, 10.0]], columns=["A", "B", "C"] ) result = parser.read_csv(StringIO(data), comment="#", skiprows=4, header=1) tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("comment_char", ["#", "~", "&", "^", "*", "@"]) def test_custom_comment_char(all_parsers, comment_char): parser = all_parsers data = "a,b,c\n1,2,3#ignore this!\n4,5,6#ignorethistoo" result = parser.read_csv( StringIO(data.replace("#", comment_char)), comment=comment_char ) expected = DataFrame([[1, 2, 3], [4, 5, 6]], columns=["a", "b", "c"]) tm.assert_frame_equal(result, expected) @pytest.mark.parametrize("header", ["infer", None]) def test_comment_first_line(all_parsers, header): # see gh-4623 parser = all_parsers data = "# notes\na,b,c\n# more notes\n1,2,3" if header is None: expected = DataFrame({0: ["a", "1"], 1: ["b", "2"], 2: ["c", "3"]}) else: expected = DataFrame([[1, 2, 3]], columns=["a", "b", "c"]) result = parser.read_csv(StringIO(data), comment="#", header=header) tm.assert_frame_equal(result, expected)