IO-471 Support for additional encodings in ReversedLinesFileReader

Patch by Leandro Reis, applied with patch adjustments to trunk

git-svn-id: https://svn.apache.org/repos/asf/commons/proper/io/trunk@1686739 13f79535-47bb-0310-9956-ffa450edef68

Author: krosenvold

src/main/java/org/apache/commons/io/input/ReversedLinesFileReader.java

@@ -121,9 +121,11 @@ public ReversedLinesFileReader(final File file, final int blockSize, final Chars
             // UTF-8 works fine out of the box, for multibyte sequences a second UTF-8 byte can never be a newline byte
             // http://en.wikipedia.org/wiki/UTF-8
             byteDecrement = 1;
-        } else if (charset == Charset.forName("Shift_JIS")) {
-            // Same as for UTF-8
-            // http://www.herongyang.com/Unicode/JIS-Shift-JIS-Encoding.html
+        } else if(charset == Charset.forName("Shift_JIS") || // Same as for UTF-8 http://www.herongyang.com/Unicode/JIS-Shift-JIS-Encoding.html
+                charset == Charset.forName("windows-31j") || // Windows code page 932 (Japanese)
+                charset == Charset.forName("x-windows-949") || // Windows code page 949 (Korean)
+                charset == Charset.forName("gbk") || // Windows code page 936 (Simplified Chinese)
+                charset == Charset.forName("x-windows-950")) { // Windows code page 950 (Traditional Chinese)
             byteDecrement = 1;
         } else if (charset == Charsets.UTF_16BE || charset == Charsets.UTF_16LE) {
             // UTF-16 new line sequences are not allowed as second tuple of four byte sequences,
@@ -356,4 +358,4 @@ private int getNewLineMatchByteCount(final byte[] data, final int i) {
         }
     }
 
-}
+}

src/test/java/org/apache/commons/io/input/ReversedLinesFileReaderTestParamBlockSize.java

@@ -59,6 +59,18 @@ public ReversedLinesFileReaderTestParamBlockSize(final Integer testWithBlockSize
     private static final String TEST_LINE_SHIFT_JIS1 = "Hiragana letters: \u3041\u3042\u3043\u3044\u3045";
     // Kanji letters: 明輸�京
     private static final String TEST_LINE_SHIFT_JIS2 = "Kanji letters: \u660E\u8F38\u5B50\u4EAC";
+    // windows-31j characters
+    private static final String TEST_LINE_WINDOWS_31J_1 = "\u3041\u3042\u3043\u3044\u3045";
+    private static final String TEST_LINE_WINDOWS_31J_2 = "\u660E\u8F38\u5B50\u4EAC";
+    // gbk characters (Simplified Chinese)
+    private static final String TEST_LINE_GBK_1 = "\u660E\u8F38\u5B50\u4EAC";
+    private static final String TEST_LINE_GBK_2 = "\u7B80\u4F53\u4E2D\u6587";
+    // x-windows-949 characters (Korean)
+    private static final String TEST_LINE_X_WINDOWS_949_1 = "\uD55C\uAD6D\uC5B4";
+    private static final String TEST_LINE_X_WINDOWS_949_2 = "\uB300\uD55C\uBBFC\uAD6D";
+    // x-windows-950 characters (Traditional Chinese)
+    private static final String TEST_LINE_X_WINDOWS_950_1 = "\u660E\u8F38\u5B50\u4EAC";
+    private static final String TEST_LINE_X_WINDOWS_950_2 = "\u7E41\u9AD4\u4E2D\u6587";
 
 
     @After
@@ -127,6 +139,38 @@ public void testShiftJISFile() throws URISyntaxException, IOException {
         assertEqualsAndNoLineBreaks(TEST_LINE_SHIFT_JIS1, reversedLinesFileReader.readLine());
     }
 
+    @Test
+    public void testWindows31jFile() throws URISyntaxException, IOException {
+        final File testFileWindows31J = new File(this.getClass().getResource("/test-file-windows-31j.bin").toURI());
+        reversedLinesFileReader = new ReversedLinesFileReader(testFileWindows31J, testParamBlockSize, "windows-31j");
+        assertEqualsAndNoLineBreaks(TEST_LINE_WINDOWS_31J_2, reversedLinesFileReader.readLine());
+        assertEqualsAndNoLineBreaks(TEST_LINE_WINDOWS_31J_1, reversedLinesFileReader.readLine());
+    }
+
+    @Test
+    public void testGBK() throws URISyntaxException, IOException {
+        final File testFileGBK = new File(this.getClass().getResource("/test-file-gbk.bin").toURI());
+        reversedLinesFileReader = new ReversedLinesFileReader(testFileGBK, testParamBlockSize, "GBK");
+        assertEqualsAndNoLineBreaks(TEST_LINE_GBK_2, reversedLinesFileReader.readLine());
+        assertEqualsAndNoLineBreaks(TEST_LINE_GBK_1, reversedLinesFileReader.readLine());
+    }
+
+    @Test
+    public void testxWindows949File() throws URISyntaxException, IOException {
+        final File testFilexWindows949 = new File(this.getClass().getResource("/test-file-x-windows-949.bin").toURI());
+        reversedLinesFileReader = new ReversedLinesFileReader(testFilexWindows949, testParamBlockSize, "x-windows-949");
+        assertEqualsAndNoLineBreaks(TEST_LINE_X_WINDOWS_949_2, reversedLinesFileReader.readLine());
+        assertEqualsAndNoLineBreaks(TEST_LINE_X_WINDOWS_949_1, reversedLinesFileReader.readLine());
+    }
+
+    @Test
+    public void testxWindows950File() throws URISyntaxException, IOException {
+        final File testFilexWindows950 = new File(this.getClass().getResource("/test-file-x-windows-950.bin").toURI());
+        reversedLinesFileReader = new ReversedLinesFileReader(testFilexWindows950, testParamBlockSize, "x-windows-950");
+        assertEqualsAndNoLineBreaks(TEST_LINE_X_WINDOWS_950_2, reversedLinesFileReader.readLine());
+        assertEqualsAndNoLineBreaks(TEST_LINE_X_WINDOWS_950_1, reversedLinesFileReader.readLine());
+    }
+
     @Test // this test is run 3x for same block size as we want to test with 10
     public void testFileSizeIsExactMultipleOfBlockSize() throws URISyntaxException, IOException {
         final int blockSize = 10;

src/test/java/org/apache/commons/io/input/ReversedLinesFileReaderTestParamFile.java

@@ -57,6 +57,10 @@ public static Collection<Object[]> blockSizes() {
                 {"test-file-utf8-win-linebr.bin", "UTF-8", 3},
                 {"test-file-utf8-win-linebr.bin", "UTF-8", 4},
                 {"test-file-utf8.bin", "UTF-8", null},
+                {"test-file-windows-31j.bin", "windows-31j", null},
+                {"test-file-gbk.bin", "gbk", null},
+                {"test-file-x-windows-949.bin", "x-windows-949", null},
+                {"test-file-x-windows-950.bin", "x-windows-950", null},
         });
     }
 

src/test/resources/test-file-gbk.bin

@@ -0,0 +1,2 @@
+��ݔ�Ӿ�
+��������

src/test/resources/test-file-windows-31j.bin

@@ -0,0 +1,2 @@
+����������
+���A�q��

src/test/resources/test-file-x-windows-949.bin

@@ -0,0 +1,2 @@
+�ѱ���
+���ѹα�

src/test/resources/test-file-x-windows-950.bin

@@ -0,0 +1,2 @@
+����l��
+�c�餤��