- Add some real world CSV files.

garydgregory · garydgregory · commit 316a51f5208b · 2013-08-01T02:03:33.000Z
- Add a record API to get column values using an Enum. - Throw a better exception when a resource is not found in a class loader for the parser. - Replace some tabs with spaces. git-svn-id: https://svn.apache.org/repos/asf/commons/proper/csv/trunk@1509068 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/main/java/org/apache/commons/csv/CSVParser.java b/src/main/java/org/apache/commons/csv/CSVParser.java
@@ -106,7 +106,7 @@ public class CSVParser implements Iterable<CSVRecord>, Closeable {
     public static CSVParser parseFile(File file, final CSVFormat format) throws IOException {
         return new CSVParser(new FileReader(file), format);
     }
-    
+
     /**
      * Creates a parser for the given resource.
      * 
@@ -128,9 +128,38 @@ public static CSVParser parseFile(File file, final CSVFormat format) throws IOEx
      */
     public static CSVParser parseResource(String resource, Charset charset, ClassLoader classLoader,
             final CSVFormat format) throws IOException {
-        return parseURL(classLoader.getResource(resource), charset, format);
+        URL url = classLoader.getResource(resource);
+        if (url == null) {
+            throw new IllegalArgumentException("Resource cannot be found: " + resource);
+        }
+        return parseURL(url, charset, format);
     }
-    
+
+    /**
+     * Creates a parser for the given resource.
+     * 
+     * <p>
+     * If you do not read all records from the given source, you should call {@link #close()} on the parser.
+     * </p>
+     * 
+     * @param resource
+     *            a resource path
+     * @param charset
+     *            the charset for the resource
+     * @param format
+     *            the CSVFormat used for CSV parsing
+     * @return a new parser
+     * @throws IOException
+     *             If an I/O error occurs
+     */
+    public static CSVParser parseResource(String resource, Charset charset, final CSVFormat format) throws IOException {
+        URL url = ClassLoader.getSystemResource(resource);
+        if (url == null) {
+            throw new IllegalArgumentException("System resource cannot be found: " + resource);
+        }
+        return parseURL(url, charset, format);
+    }
+
     /**
      * Creates a parser for the given {@link String} using the default format {@link CSVFormat#DEFAULT}.
      * 
@@ -201,7 +230,7 @@ public static CSVParser parseURL(URL url, Charset charset, final CSVFormat forma
 
     /**
      * CSV parser using the default format {@link CSVFormat#DEFAULT}.
-     *
+     * 
      * <p>
      * If you do not read all records from the given {@code reader}, you should call {@link #close()} on the parser,
      * unless you close the {@code reader}.
@@ -249,25 +278,26 @@ private void addRecordValue() {
             this.record.add(input);
         } else {
             this.record.add(input.equalsIgnoreCase(nullString) ? null : input);
-        }}
+        }
+    }
 
     /**
      * Closes resources.
      * 
-	 * @throws IOException
-	 *             If an I/O error occurs
+     * @throws IOException
+     *             If an I/O error occurs
      */
-	public void close() throws IOException {
-		if (this.lexer != null) {
-			this.lexer.close();
-		}		
-	}
+    public void close() throws IOException {
+        if (this.lexer != null) {
+            this.lexer.close();
+        }
+    }
 
     /**
      * Returns the current line number in the input stream.
      * <p/>
      * ATTENTION: If your CSV input has multi-line values, the returned number does not correspond to the record number.
-     *
+     * 
      * @return current line number
      */
     public long getCurrentLineNumber() {
@@ -277,9 +307,8 @@ public long getCurrentLineNumber() {
     /**
      * Returns a copy of the header map that iterates in column order.
      * <p>
-     * The map keys are column names.
-     * The map values are 0-based indices.
-     *
+     * The map keys are column names. The map values are 0-based indices.
+     * 
      * @return a copy of the header map that iterates in column order.
      */
     public Map<String, Integer> getHeaderMap() {
@@ -290,7 +319,7 @@ public Map<String, Integer> getHeaderMap() {
      * Returns the current record number in the input stream.
      * <p/>
      * ATTENTION: If your CSV input has multi-line values, the returned number does not correspond to the line number.
-     *
+     * 
      * @return current line number
      */
     public long getRecordNumber() {
@@ -302,7 +331,7 @@ public long getRecordNumber() {
      * entries.
      * <p/>
      * The returned content starts at the current parse-position in the stream.
-     *
+     * 
      * @return list of {@link CSVRecord} entries, may be empty
      * @throws IOException
      *             on parse error or input read-failure
@@ -350,10 +379,10 @@ private Map<String, Integer> initializeHeader() throws IOException {
     }
 
     public boolean isClosed() {
-		return this.lexer.isClosed();
-	}
+        return this.lexer.isClosed();
+    }
 
-	/**
+    /**
      * Returns an iterator on the records. IOExceptions occurring during the iteration are wrapped in a
      * RuntimeException.
      */
@@ -371,9 +400,9 @@ private CSVRecord getNextRecord() {
             }
 
             public boolean hasNext() {
-            	if (CSVParser.this.isClosed()) {
-            		return false;
-            	}
+                if (CSVParser.this.isClosed()) {
+                    return false;
+                }
                 if (this.current == null) {
                     this.current = this.getNextRecord();
                 }
@@ -382,9 +411,9 @@ public boolean hasNext() {
             }
 
             public CSVRecord next() {
-            	if (CSVParser.this.isClosed()) {
-            		return null;
-            	}
+                if (CSVParser.this.isClosed()) {
+                    return null;
+                }
                 CSVRecord next = this.current;
                 this.current = null;
 
@@ -407,7 +436,7 @@ public void remove() {
 
     /**
      * Parses the next record from the current point in the stream.
-     *
+     * 
      * @return the record as an array of values, or <tt>null</tt> if the end of the stream has been reached
      * @throws IOException
      *             on parse error or input read-failure
@@ -448,7 +477,8 @@ CSVRecord nextRecord() throws IOException {
         if (!this.record.isEmpty()) {
             this.recordNumber++;
             final String comment = sb == null ? null : sb.toString();
-            result = new CSVRecord(this.record.toArray(new String[this.record.size()]), this.headerMap, comment, this.recordNumber);
+            result = new CSVRecord(this.record.toArray(new String[this.record.size()]), this.headerMap, comment,
+                    this.recordNumber);
         }
         return result;
     }
diff --git a/src/main/java/org/apache/commons/csv/CSVRecord.java b/src/main/java/org/apache/commons/csv/CSVRecord.java
@@ -53,6 +53,17 @@ public class CSVRecord implements Serializable, Iterable<String> {
         this.comment = comment;
     }
 
+    /**
+     * Returns a value by {@link Enum}.
+     *
+     * @param e
+     *            an enum
+     * @return the String at the given enum String
+     */
+    public String get(Enum<?> e) {
+        return get(e.toString());
+    }
+
     /**
      * Returns a value by index.
      *
@@ -171,4 +182,5 @@ public String toString() {
         return Arrays.toString(values);
     }
 
+
 }
diff --git a/src/test/java/org/apache/commons/csv/FercGovTest.java b/src/test/java/org/apache/commons/csv/FercGovTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.commons.csv;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.List;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Real world examples from http://www.ferc.gov/docs-filing/eqr/soft-tools/sample-csv.asp
+ */
+public class FercGovTest {
+
+    private enum ContractColumnNames {
+        contract_id, seller_company_name, customer_company_name, customer_duns_number, contract_affiliate,
+        FERC_tariff_reference, contract_service_agreement_id, contract_execution_date, contract_commencement_date,
+        contract_termination_date, actual_termination_date, extension_provision_description, class_name, term_name,
+        increment_name, increment_peaking_name, product_type_name, product_name, quantity, units_for_contract, rate,
+        rate_minimum, rate_maximum, rate_description, units_for_rate, point_of_receipt_control_area,
+        point_of_receipt_specific_location, point_of_delivery_control_area, point_of_delivery_specific_location,
+        begin_date, end_date, time_zone;
+    }
+
+    private static final Charset US_ASCII = Charset.forName("US-ASCII");
+
+    @Test
+    public void testContractFile() throws IOException {
+        final CSVParser parser = CSVParser.parseResource("ferc.gov/contract.txt", US_ASCII,
+                CSVFormat.DEFAULT.withHeader());
+        try {
+            final List<CSVRecord> records = parser.getRecords();
+            CSVRecord record = records.get(0);
+            Assert.assertEquals(22, records.size());
+            // first record
+            Assert.assertEquals("C71", record.get(ContractColumnNames.contract_id));
+            Assert.assertEquals("The Electric Company", record.get(ContractColumnNames.seller_company_name));
+            Assert.assertEquals("ES", record.get(ContractColumnNames.time_zone));
+            // last record
+            record = records.get(records.size() - 1);
+            // first record
+            Assert.assertEquals("C78", record.get(ContractColumnNames.contract_id));
+            Assert.assertEquals("The Electric Company", record.get(ContractColumnNames.seller_company_name));
+            Assert.assertEquals("EP", record.get(ContractColumnNames.time_zone));
+        } finally {
+            parser.close();
+        }
+    }
+
+    @Test
+    public void testTransactionFile() throws IOException {
+        final CSVParser parser = CSVParser.parseResource("ferc.gov/transaction.txt", US_ASCII,
+                CSVFormat.DEFAULT.withHeader());
+        try {
+            final List<CSVRecord> records = parser.getRecords();
+            Assert.assertEquals(24, records.size());
+            CSVRecord record = records.get(0);
+            // first record
+            Assert.assertEquals("T1", record.get("transaction_unique_identifier"));
+            Assert.assertEquals("The Electric Company", record.get("seller_company_name"));
+            Assert.assertEquals("880386", record.get("transaction_charge"));
+            // last record
+            record = records.get(records.size() - 1);
+            Assert.assertEquals("T15", record.get("transaction_unique_identifier"));
+            Assert.assertEquals("The Electric Company", record.get("seller_company_name"));
+            Assert.assertEquals("1800", record.get("transaction_charge"));
+        } finally {
+            parser.close();
+        }
+    }
+}
diff --git a/src/test/resources/ferc.gov/contract.txt b/src/test/resources/ferc.gov/contract.txt
@@ -0,0 +1,23 @@
+contract_id,seller_company_name,customer_company_name,customer_duns_number,contract_affiliate,FERC_tariff_reference,contract_service_agreement_id,contract_execution_date,contract_commencement_date,contract_termination_date,actual_termination_date,extension_provision_description,class_name,term_name,increment_name,increment_peaking_name,product_type_name,product_name,quantity,units_for_contract,rate,rate_minimum,rate_maximum,rate_description,units_for_rate,point_of_receipt_control_area,point_of_receipt_specific_location,point_of_delivery_control_area,point_of_delivery_specific_location,begin_date,end_date,time_zone
+C71,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Original Volume No. 10,2,2/15/2001,2/15/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,,,,,,ES
+C72,The Electric Company,Utility A,38495837,n,FERC Electric Tariff Original Volume No. 10,15,7/25/2001,8/1/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,,,,,,ES
+C73,The Electric Company,Utility B,493758794,N,FERC Electric Tariff Original Volume No. 10,7,6/8/2001,7/6/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,, , ,,,ep
+C74,The Electric Company,Utility C,594739573,n,FERC Electric Tariff Original Volume No. 10,25,6/8/2001,7/6/2001,,,Evergreen,N/A,N/A,N/A,N/A,MB,ENERGY,0,, , , ,Market Based,,,, , ,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,ENERGY,2000,KWh,.1475, , ,Max amount of capacity and energy to be transmitted.  Bill based on monthly max delivery to City.,$/KWh,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,point-to-point agreement,2000,KW,0.01, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,network,2000,KW,0.2, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,BLACK START SERVICE,2000,KW,0.22, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,CAPACITY,2000,KW,0.04, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,regulation & frequency response,2000,KW,0.1, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C75,The Electric Company,The Power Company,456543333,N,FERC Electric Tariff Third Revised Volume No. 7,94,2/13/2001,7/1/2001,12/31/2006,,None,F,LT,M,P,T,real power transmission loss,2000,KW,7, , ,,$/kw-mo,PJM,Point A,PJM,Point B,,,ep
+C76,The Electric Company,The Power Company,456534333,N,FERC Electric Tariff Original Volume No. 10,132,12/15/2001,1/1/2002,12/31/2004,12/31/2004,None,F,LT,M,FP,MB,CAPACITY,70,MW,3750, , ,70MW for each and every hour over the term of the agreement (7x24 schedule).,$/MW,,,,,,,ep
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,35, , ,,$/MWH,,,PJM,Bus 4321,20020101,20030101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,37, , ,,$/MWH,,,PJM,Bus 4321,20030101,20040101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,39, , ,,$/MWH,,,PJM,Bus 4321,20040101,20050101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,41, , ,,$/MWH,,,PJM,Bus 4321,20050101,20060101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,43, , ,,$/MWH,,,PJM,Bus 4321,20060101,20070101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,45, , ,,$/MWH,,,PJM,Bus 4321,20070101,20080101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,47, , ,,$/MWH,,,PJM,Bus 4321,20080101,20090101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,49, , ,,$/MWH,,,PJM,Bus 4321,20090101,20100101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,51, , ,,$/MWH,,,PJM,Bus 4321,20100101,20110101,EP
+C78,The Electric Company,"The Electric Marketing Co., LLC",23456789,Y,FERC Electric Tariff Original Volume No. 2,Service Agreement 1,1/2/1992,1/2/1992,1/1/2012,,Renewable annually by mutual agreement after termination date.,UP,LT,Y,FP,CB,ENERGY,0,MWH,53, , ,,$/MWH,,,PJM,Bus 4321,20110101,20120101,EP
diff --git a/src/test/resources/ferc.gov/readme.txt b/src/test/resources/ferc.gov/readme.txt
@@ -0,0 +1 @@
+Real world examples from http://www.ferc.gov/docs-filing/eqr/soft-tools/sample-csv.asp
diff --git a/src/test/resources/ferc.gov/transaction.txt b/src/test/resources/ferc.gov/transaction.txt

Original file line number	Diff line number	Diff line change
`@@ -53,6 +53,17 @@ public class CSVRecord implements Serializable, Iterable<String> {`
`53`	`53`	`this.comment = comment;`
`54`	`54`	`}`
`55`	`55`
	`56`	`+ /**`
	`57`	`+ * Returns a value by {@link Enum}.`
	`58`	`+ *`
	`59`	`+ * @param e`
	`60`	`+ * an enum`
	`61`	`+ * @return the String at the given enum String`
	`62`	`+ */`
	`63`	`+ public String get(Enum<?> e) {`
	`64`	`+ return get(e.toString());`
	`65`	`+ }`
	`66`	`+`
`56`	`67`	`/**`
`57`	`68`	`* Returns a value by index.`
`58`	`69`	`*`
`@@ -171,4 +182,5 @@ public String toString() {`
`171`	`182`	`return Arrays.toString(values);`
`172`	`183`	`}`
`173`	`184`
	`185`	`+`
`174`	`186`	`}`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Real world examples from http://www.ferc.gov/docs-filing/eqr/soft-tools/sample-csv.asp`