IO-354 Commons IO Tailer does not respect UTF-8 Charset

sebbASF · sebbASF · commit 8d52f694da04 · 2013-04-17T00:59:42.000Z
git-svn-id: https://svn.apache.org/repos/asf/commons/proper/io/trunk@1468703 13f79535-47bb-0310-9956-ffa450edef68
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
@@ -47,6 +47,9 @@ The <action> type attribute can be add,update,fix,remove.
   <body>
     <!-- The release date is the date RC is cut -->
     <release version="2.5" date="2013-??-??" description="New features and bug fixes.">    
+      <action issue="IO-354" dev="sebb" type="fix">
+        Commons IO Tailer does not respect UTF-8 Charset.
+      </action>            
       <action issue="IO-323" dev="sebb" type="fix">
         What should happen in FileUtils.sizeOf[Directory] when an overflow takes place?
         Added Javadoc. 
diff --git a/src/main/java/org/apache/commons/io/input/Tailer.java b/src/main/java/org/apache/commons/io/input/Tailer.java
@@ -16,10 +16,12 @@
  */
 package org.apache.commons.io.input;
 
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
+import java.nio.charset.Charset;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
@@ -444,11 +446,11 @@ public void stop() {
      * @throws java.io.IOException if an I/O error occurs.
      */
     private long readLines(final RandomAccessFile reader) throws IOException {
-        final StringBuilder sb = new StringBuilder();
-
+        // Make explicit that the default charset is being used here
+        Charset cset = Charset.defaultCharset();
+        ByteArrayOutputStream lineBuf = new ByteArrayOutputStream(64);
         long pos = reader.getFilePointer();
         long rePos = pos; // position to re-read
-
         int num;
         boolean seenCR = false;
         while (getRun() && ((num = reader.read(inbuf)) != -1)) {
@@ -457,30 +459,29 @@ private long readLines(final RandomAccessFile reader) throws IOException {
                 switch (ch) {
                 case '\n':
                     seenCR = false; // swallow CR before LF
-                    listener.handle(sb.toString());
-                    sb.setLength(0);
+                    listener.handle(new String(lineBuf.toByteArray(), cset));
+                    lineBuf.reset();
                     rePos = pos + i + 1;
                     break;
                 case '\r':
                     if (seenCR) {
-                        sb.append('\r');
+                        lineBuf.write('\r');
                     }
                     seenCR = true;
                     break;
                 default:
                     if (seenCR) {
                         seenCR = false; // swallow final CR
-                        listener.handle(sb.toString());
-                        sb.setLength(0);
+                        listener.handle(new String(lineBuf.toByteArray(), cset));
+                        lineBuf.reset(); 
                         rePos = pos + i + 1;
                     }
-                    sb.append((char) ch); // add character, not its ascii value
+                    lineBuf.write(ch);
                 }
             }
-
             pos = reader.getFilePointer();
         }
-
+        IOUtils.closeQuietly(lineBuf); // not strictly necessary
         reader.seek(rePos); // Ensure we can re-read if necessary
         return rePos;
     }
diff --git a/src/test/java/org/apache/commons/io/input/TailerTest.java b/src/test/java/org/apache/commons/io/input/TailerTest.java
@@ -16,10 +16,13 @@
  */
 package org.apache.commons.io.input;
 
+import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.RandomAccessFile;
 import java.io.Writer;
 import java.util.ArrayList;
@@ -105,6 +108,42 @@ public void testBufferBreak() throws Exception {
         listener.clear();
     }
 
+    public void testMultiByteBreak() throws Exception {
+        final long delay = 50;
+        final File origin = new File(this.getClass().getResource("/test-file-utf8.bin").toURI());
+        final File file = new File(getTestDirectory(), "testMultiByteBreak.txt");
+        createFile(file, 0);
+        final TestTailerListener listener = new TestTailerListener();
+        final String osname = System.getProperty("os.name");
+        final boolean isWindows = osname.startsWith("Windows");
+        tailer = new Tailer(file, listener, delay, false, isWindows);
+        final Thread thread = new Thread(tailer);
+        thread.start();
+		
+        
+        BufferedReader reader = null;
+        try{
+        	List<String> lines = new ArrayList<String>();
+        	reader = new BufferedReader(new InputStreamReader(new FileInputStream(origin)));
+        	String line = null;
+        	while((line = reader.readLine()) != null){
+        		write(file, line);
+        		lines.add(line);
+        	}
+
+           final long testDelayMillis = delay * 10;
+           Thread.sleep(testDelayMillis);
+           List<String> tailerlines = listener.getLines();
+           assertEquals("line count",lines.size(),tailerlines.size());
+           for(int i = 0,len = lines.size();i<len;i++){
+        	   assertEquals("line "+i, lines.get(i), tailerlines.get(i));
+           }
+        }finally{
+        	tailer.stop();
+        	IOUtils.closeQuietly(reader);
+        }
+    }
+
     public void testTailerEof() throws Exception {
         // Create & start the Tailer
         final long delay = 50;
