Made line comments work with whitespace delim and custom line terminator

Author: mdmueller

pandas/io/tests/test_parsers.py

@@ -2944,6 +2944,14 @@ def test_line_comment(self):
                     [5., np.nan, 10.]]
         df = self.read_csv(StringIO(data), comment='#')
         tm.assert_almost_equal(df.values, expected)
+        # check with delim_whitespace=True
+        df = self.read_csv(StringIO(data.replace(',', ' ')), comment='#',
+                            delim_whitespace=True)
+        tm.assert_almost_equal(df.values, expected)
+        # check with custom line terminator
+        df = self.read_csv(StringIO(data.replace('\n', '*')), comment='#',
+                            lineterminator='*')
+        tm.assert_almost_equal(df.values, expected)
 
     def test_comment_skiprows(self):
         data = """# empty

pandas/src/parser/tokenizer.c

@@ -969,6 +969,10 @@ int tokenize_delim_customterm(parser_t *self, size_t line_limit)
                 END_LINE();
                 break;
             }
+            else if (c == self->commentchar) {
+                self->state = EAT_LINE_COMMENT;
+                break;
+            }
             /* normal character - handle as START_FIELD */
             self->state = START_FIELD;
             /* fallthru */
@@ -1103,6 +1107,13 @@ int tokenize_delim_customterm(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_LINE_COMMENT:
+            if (c == self->lineterminator) {
+                self->file_lines++;
+                self->state = START_RECORD;
+            }
+            break;
+
         case EAT_COMMENT:
             if (c == self->lineterminator) {
                 END_LINE();
@@ -1186,6 +1197,9 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
             } else if (IS_WHITESPACE(c)) {
                 self->state = EAT_WHITESPACE;
                 break;
+            } else if (c == self->commentchar) {
+                self->state = EAT_LINE_COMMENT;
+                break;
             } else {
                 /* normal character - handle as START_FIELD */
                 self->state = START_FIELD;
@@ -1231,6 +1245,16 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_LINE_COMMENT:
+            if (c == '\n') {
+                self->file_lines++;
+                self->state = START_RECORD;
+            } else if (c == '\r') {
+                self->file_lines++;
+                self->state = EAT_CRNL_NOP;
+            }
+            break;
+
         case ESCAPED_CHAR:
             /* if (c == '\0') */
             /*  c = '\n'; */
@@ -1351,6 +1375,15 @@ int tokenize_whitespace(parser_t *self, size_t line_limit)
             }
             break;
 
+        case EAT_CRNL_NOP: // inside an ignored comment line
+            self->state = START_RECORD;
+            /* \r line terminator -- parse this character again */
+            if (c != '\n' && c != self->delimiter) {
+                --i;
+                --buf;
+            }
+            break;
+
         case EAT_COMMENT:
             if (c == '\n') {
                 END_LINE();