[IO-597] FileUtils.iterateFiles runs out of memory when executed for a

directory with large number of files.

Re-implement FileUtils' iterateFiles(), iterateFilesAndDirs(),
listFiles(), listFilesAndDirs() to use NIO file tree walking instead of
IO file listings to avoid memory consumption issues on large file trees.

Author: Gary Gregory

src/changes/changes.xml

@@ -66,6 +66,11 @@ The <action> type attribute can be add,update,fix,remove.
       <action dev="ggregory" type="fix" due-to="Gary Gregory">
         AccumulatorPathVisitor does not track directories properly.
       </action>
+      <action issue="IO-597" dev="ggregory" type="fix" due-to="Gary Gregory, Arvind, Rob Spoor">
+        FileUtils.iterateFiles runs out of memory when executed for a directory with large number of files.
+        Re-implement FileUtils' iterateFiles(), iterateFilesAndDirs(), listFiles(), listFilesAndDirs() to use NIO 
+        file tree walking instead of IO file listings to avoid memory consumption issues on large file trees.
+      </action>
       <!-- ADD -->
       <action dev="ggregory" type="add" due-to="Gary Gregory">
         Add FileSystemProviders class.

src/main/java/org/apache/commons/io/FileUtils.java

@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.io;
+
+import java.io.Closeable;
+import java.util.Iterator;
+import java.util.Objects;
+import java.util.stream.Stream;
+
+/**
+ * Wraps and presents a stream as a closable iterator resource that automatically closes itself when reaching the end
+ * of stream.
+ *
+ * @param <E> The stream and iterator type.
+ * @since 2.9.0
+ */
+class StreamIterator<E> implements Iterator<E>, Closeable {
+
+    /**
+     * Wraps and presents a stream as a closable resource that automatically closes itself when reaching the end of
+     * stream.
+     * <h2>Warning</h2>
+     * <p>
+     * In order to close the stream, the call site MUST either close the stream it allocated OR call the iterator until
+     * the end.
+     * </p>
+     * 
+     * @param <T> The stream and iterator type.
+     * @param stream The stream iterate.
+     * @return A new iterator.
+     */
+    @SuppressWarnings("resource") // Caller MUST close or iterate to the end.
+    public static <T> Iterator<T> iterator(final Stream<T> stream) {
+        return new StreamIterator<>(stream).iterator;
+    }
+
+    private StreamIterator(final Stream<E> stream) {
+        super();
+        this.stream = Objects.requireNonNull(stream, "stream");
+        this.iterator = stream.iterator();
+    }
+
+    private final Iterator<E> iterator;
+    private final Stream<E> stream;
+
+    @Override
+    public boolean hasNext() {
+        final boolean hasNext = iterator.hasNext();
+        if (!hasNext) {
+            close();
+        }
+        return hasNext;
+    }
+
+    @Override
+    public E next() {
+        final E next = iterator.next();
+        if (next == null) {
+            close();
+        }
+        return next;
+    }
+
+    /**
+     * Closes the underlying stream.
+     */
+    @Override
+    public void close() {
+        stream.close();
+
+    }
+
+}

src/main/java/org/apache/commons/io/StreamIterator.java

@@ -35,7 +35,7 @@
  * <h2>Example</h2>
  *
  * <pre>
- * Path dir = Paths.get(".");
+ * Path dir = Paths.get("");
  * // We are interested in files older than one day
  * long cutoff = System.currentTimeMillis() - (24 * 60 * 60 * 1000);
  * AccumulatorPathVisitor visitor = AccumulatorPathVisitor.withLongCounters(new AgeFileFilter(cutoff));
@@ -70,12 +70,14 @@ public static AccumulatorPathVisitor withBigIntegerCounters() {
     /**
      * Creates a new instance configured with a BigInteger {@link PathCounters}.
      *
-     * @param pathFilter Filters files to accumulate and count.
+     * @param fileFilter Filters files to accumulate and count.
+     * @param dirFilter Filters directories to accumulate and count.
      * @return a new instance configured with a long {@link PathCounters}.
      * @since 2.9.0
      */
-    public static AccumulatorPathVisitor withBigIntegerCounters(final PathFilter pathFilter) {
-        return new AccumulatorPathVisitor(Counters.bigIntegerPathCounters(), pathFilter);
+    public static AccumulatorPathVisitor withBigIntegerCounters(final PathFilter fileFilter,
+        final PathFilter dirFilter) {
+        return new AccumulatorPathVisitor(Counters.bigIntegerPathCounters(), fileFilter, dirFilter);
     }
 
     /**
@@ -90,12 +92,13 @@ public static AccumulatorPathVisitor withLongCounters() {
     /**
      * Creates a new instance configured with a long {@link PathCounters}.
      *
-     * @param pathFilter Filters files to accumulate and count.
+     * @param fileFilter Filters files to accumulate and count.
+     * @param dirFilter Filters directories to accumulate and count.
      * @return a new instance configured with a long {@link PathCounters}.
      * @since 2.9.0
      */
-    public static AccumulatorPathVisitor withLongCounters(final PathFilter pathFilter) {
-        return new AccumulatorPathVisitor(Counters.longPathCounters(), pathFilter);
+    public static AccumulatorPathVisitor withLongCounters(final PathFilter fileFilter, final PathFilter dirFilter) {
+        return new AccumulatorPathVisitor(Counters.longPathCounters(), fileFilter, dirFilter);
     }
 
     private final List<Path> dirList = new ArrayList<>();
@@ -124,11 +127,13 @@ public AccumulatorPathVisitor(final PathCounters pathCounter) {
      * Constructs a new instance.
      *
      * @param pathCounter How to count path visits.
-     * @param pathFilter Filters which paths to count.
+     * @param pathFilter Filters which files to count.
+     * @param dirFilter Filters which directories to count.
      * @since 2.9.0
      */
-    public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter pathFilter) {
-        super(pathCounter, pathFilter);
+    public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter pathFilter,
+        final PathFilter dirFilter) {
+        super(pathCounter, pathFilter, dirFilter);
     }
 
     private void add(final List<Path> list, final Path dir) {

src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java

@@ -59,15 +59,16 @@ public CopyDirectoryVisitor(final PathCounters pathCounter, final Path sourceDir
      * Constructs a new visitor that deletes files except for the files and directories explicitly given.
      *
      * @param pathCounter How to count visits.
-     * @param pathFilter How to filter paths.
+     * @param fileFilter How to filter file paths.
+     * @param dirFilter How to filter directory paths.
      * @param sourceDirectory The source directory
      * @param targetDirectory The target directory
      * @param copyOptions Specifies how the copying should be done.
      * @since 2.9.0
      */
-    public CopyDirectoryVisitor(final PathCounters pathCounter, final PathFilter pathFilter, final Path sourceDirectory, final Path targetDirectory,
-        final CopyOption... copyOptions) {
-        super(pathCounter, pathFilter);
+    public CopyDirectoryVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+        final Path sourceDirectory, final Path targetDirectory, final CopyOption... copyOptions) {
+        super(pathCounter, fileFilter, dirFilter);
         this.sourceDirectory = sourceDirectory;
         this.targetDirectory = targetDirectory;
         this.copyOptions = copyOptions == null ? PathUtils.EMPTY_COPY_OPTIONS : copyOptions.clone();

src/main/java/org/apache/commons/io/file/CopyDirectoryVisitor.java

@@ -55,28 +55,31 @@ public static CountingPathVisitor withLongCounters() {
     }
 
     private final PathCounters pathCounters;
-    private final PathFilter pathFilter;
+    private final PathFilter fileFilter;
+    private final PathFilter dirFilter;
 
     /**
      * Constructs a new instance.
      *
      * @param pathCounter How to count path visits.
      */
     public CountingPathVisitor(final PathCounters pathCounter) {
-        this(pathCounter, TrueFileFilter.INSTANCE);
+        this(pathCounter, TrueFileFilter.INSTANCE, TrueFileFilter.INSTANCE);
     }
 
     /**
      * Constructs a new instance.
      *
      * @param pathCounter How to count path visits.
-     * @param pathFilter Filters which paths to count.
+     * @param fileFilter Filters which files to count.
+     * @param dirFilter Filters which directories to count.
      * @since 2.9.0
      */
-    public CountingPathVisitor(final PathCounters pathCounter, final PathFilter pathFilter) {
+    public CountingPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, PathFilter dirFilter) {
         super();
         this.pathCounters = Objects.requireNonNull(pathCounter, "pathCounter");
-        this.pathFilter = Objects.requireNonNull(pathFilter, "pathFilter");
+        this.fileFilter = Objects.requireNonNull(fileFilter, "fileFilter");
+        this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
     }
 
     @Override
@@ -110,6 +113,12 @@ public FileVisitResult postVisitDirectory(final Path dir, final IOException exc)
         updateDirCounter(dir, exc);
         return FileVisitResult.CONTINUE;
     }
+    
+    @Override
+    public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attributes) throws IOException {
+        final FileVisitResult accept = dirFilter.accept(dir, attributes);
+        return accept != FileVisitResult.CONTINUE ? FileVisitResult.SKIP_SUBTREE : FileVisitResult.CONTINUE;
+    }
 
     @Override
     public String toString() {
@@ -140,7 +149,7 @@ protected void updateFileCounters(final Path file, final BasicFileAttributes att
 
     @Override
     public FileVisitResult visitFile(final Path file, final BasicFileAttributes attributes) throws IOException {
-        if (Files.exists(file) && pathFilter.accept(file, attributes) == FileVisitResult.CONTINUE) {
+        if (Files.exists(file) && fileFilter.accept(file, attributes) == FileVisitResult.CONTINUE) {
             updateFileCounters(file, attributes);
         }
         return FileVisitResult.CONTINUE;

src/main/java/org/apache/commons/io/file/CountingPathVisitor.java

@@ -33,7 +33,7 @@ public interface PathFilter {
      * Tests whether or not to include the specified Path in a result.
      *
      * @param path The Path to test.
-     * @param attributes the file's basic attributes.
+     * @param attributes the file's basic attributes (TODO may be null).
      * @return a FileVisitResult
      */
     FileVisitResult accept(Path path, BasicFileAttributes attributes);

src/main/java/org/apache/commons/io/file/PathFilter.java

@@ -280,6 +280,17 @@ public static PathCounters countDirectory(final Path directory) throws IOExcepti
         return visitFileTree(new CountingPathVisitor(Counters.longPathCounters()), directory).getPathCounters();
     }
 
+    /**
+     * Gets the current directory.
+     * 
+     * @return the current directory.
+     *
+     * @since 2.9.0
+     */
+    public static Path current() {
+        return Paths.get("");
+    }
+    
     /**
      * Deletes a file or directory. If the path is a directory, delete it and all sub-directories.
      * <p>
@@ -684,7 +695,8 @@ public static boolean isNewer(final Path file, final long timeMillis, final Link
      * @return a new instance.
      * @throws IOException if an I/O error occurs.
      */
-    public static DirectoryStream<Path> newDirectoryStream(final Path dir, final PathFilter pathFilter) throws IOException {
+    public static DirectoryStream<Path> newDirectoryStream(final Path dir, final PathFilter pathFilter)
+        throws IOException {
         return Files.newDirectoryStream(dir, new DirectoryStreamFilter(pathFilter));
     }
 
@@ -874,6 +886,24 @@ public static <T extends FileVisitor<? super Path>> T visitFileTree(final T visi
         return visitFileTree(visitor, Paths.get(uri));
     }
 
+    /**
+     * Returns a stream of filtered paths.
+     *
+     * @param start the start path
+     * @param pathFilter the path filter
+     * @param maxDepth the maximum depth of directories to walk.
+     * @param readAttributes whether to call the filters with file attributes (false passes null).
+     * @param options the options to configure the walk.
+     * @return a filtered stream of paths.
+     * @throws IOException if an I/O error is thrown when accessing the starting file.
+     * @since 2.9.0
+     */
+    public static Stream<Path> walk(final Path start, final PathFilter pathFilter, final int maxDepth,
+        boolean readAttributes, final FileVisitOption... options) throws IOException {
+        return Files.walk(start, maxDepth, options).filter(path -> pathFilter.accept(path,
+            readAttributes ? readBasicFileAttributesQuietly(path) : null) == FileVisitResult.CONTINUE);
+    }
+
     /**
      * Does allow to instantiate.
      */

src/main/java/org/apache/commons/io/file/PathUtils.java

@@ -39,7 +39,7 @@
  */
 public abstract class AbstractFileFilter implements IOFileFilter, PathVisitor {
 
-    protected static FileVisitResult toFileVisitResult(final boolean accept) {
+    static FileVisitResult toFileVisitResult(final boolean accept, final Path path) {
         return accept ? FileVisitResult.CONTINUE : FileVisitResult.TERMINATE;
     }
 

src/main/java/org/apache/commons/io/filefilter/AbstractFileFilter.java

@@ -34,7 +34,7 @@
  * </p>
  * <h2>Using Classic IO</h2>
  * <pre>
- * Path dir = Paths.get(".");
+ * Path dir = Paths.get("");
  * // We are interested in files older than one day
  * long cutoff = System.currentTimeMillis() - (24 * 60 * 60 * 1000);
  * String[] files = dir.list(new AgeFileFilter(cutoff));
@@ -45,7 +45,7 @@
  *
  * <h2>Using NIO</h2>
  * <pre>
- * Path dir = Paths.get(".");
+ * Path dir = Paths.get("");
  * // We are interested in files older than one day
  * long cutoff = System.currentTimeMillis() - (24 * 60 * 60 * 1000);
  * AccumulatorPathVisitor visitor = AccumulatorPathVisitor.withLongCounters(new AgeFileFilter(cutoff));
@@ -182,7 +182,7 @@ public FileVisitResult accept(final Path file, final BasicFileAttributes attribu
         } catch (final IOException e) {
             return handle(e);
         }
-        return toFileVisitResult(acceptOlder != newer);
+        return toFileVisitResult(acceptOlder != newer, file);
     }
 
     /**