    001/*
    002 *  Licensed to the Apache Software Foundation (ASF) under one or more
    003 *  contributor license agreements.  See the NOTICE file distributed with
    004 *  this work for additional information regarding copyright ownership.
    005 *  The ASF licenses this file to You under the Apache License, Version 2.0
    006 *  (the "License"); you may not use this file except in compliance with
    007 *  the License.  You may obtain a copy of the License at
    008 *
    009 *      http://www.apache.org/licenses/LICENSE-2.0
    010 *
    011 *  Unless required by applicable law or agreed to in writing, software
    012 *  distributed under the License is distributed on an "AS IS" BASIS,
    013 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    014 *  See the License for the specific language governing permissions and
    015 *  limitations under the License.
    016 *
    017 */
    018package org.apache.commons.compress.archivers.zip;
    019
    020import org.apache.commons.compress.parallel.FileBasedScatterGatherBackingStore;
    021import org.apache.commons.compress.parallel.InputStreamSupplier;
    022import org.apache.commons.compress.parallel.ScatterGatherBackingStore;
    023import org.apache.commons.compress.parallel.ScatterGatherBackingStoreSupplier;
    024
    025import java.io.File;
    026import java.io.IOException;
    027import java.util.ArrayList;
    028import java.util.List;
    029import java.util.concurrent.Callable;
    030import java.util.concurrent.ExecutionException;
    031import java.util.concurrent.ExecutorService;
    032import java.util.concurrent.Executors;
    033import java.util.concurrent.Future;
    034import java.util.concurrent.TimeUnit;
    035import java.util.concurrent.atomic.AtomicInteger;
    036import java.util.zip.Deflater;
    037
    038import static java.util.Collections.synchronizedList;
    039import static org.apache.commons.compress.archivers.zip.ZipArchiveEntryRequest.createZipArchiveEntryRequest;
    040
    041/**
    042 * Creates a zip in parallel by using multiple threadlocal {@link ScatterZipOutputStream} instances.
    043 * <p>
    044 * Note that this class generally makes no guarantees about the order of things written to
    045 * the output file. Things that need to come in a specific order (manifests, directories)
    046 * must be handled by the client of this class, usually by writing these things to the
    047 * {@link ZipArchiveOutputStream} <em>before</em> calling {@link #writeTo writeTo} on this class.</p>
    048 * <p>
    049 * The client can supply an {@link java.util.concurrent.ExecutorService}, but for reasons of
    050 * memory model consistency, this will be shut down by this class prior to completion.
    051 * </p>
    052 * @since 1.10
    053 */
    054public class ParallelScatterZipCreator {
    055    private final List<ScatterZipOutputStream> streams = synchronizedList(new ArrayList<ScatterZipOutputStream>());
    056    private final ExecutorService es;
    057    private final ScatterGatherBackingStoreSupplier backingStoreSupplier;
    058    private final List<Future<Object>> futures = new ArrayList<Future<Object>>();
    059
    060    private final long startedAt = System.currentTimeMillis();
    061    private long compressionDoneAt = 0;
    062    private long scatterDoneAt;
    063
    064    private static class DefaultBackingStoreSupplier implements ScatterGatherBackingStoreSupplier {
    065        final AtomicInteger storeNum = new AtomicInteger(0);
    066
    067        @Override
    068        public ScatterGatherBackingStore get() throws IOException {
    069            final File tempFile = File.createTempFile("parallelscatter", "n" + storeNum.incrementAndGet());
    070            return new FileBasedScatterGatherBackingStore(tempFile);
    071        }
    072    }
    073
    074    private ScatterZipOutputStream createDeferred(final ScatterGatherBackingStoreSupplier scatterGatherBackingStoreSupplier)
    075            throws IOException {
    076        final ScatterGatherBackingStore bs = scatterGatherBackingStoreSupplier.get();
    077        final StreamCompressor sc = StreamCompressor.create(Deflater.DEFAULT_COMPRESSION, bs);
    078        return new ScatterZipOutputStream(bs, sc);
    079    }
    080
    081    private final ThreadLocal<ScatterZipOutputStream> tlScatterStreams = new ThreadLocal<ScatterZipOutputStream>() {
    082        @Override
    083        protected ScatterZipOutputStream initialValue() {
    084            try {
    085                final ScatterZipOutputStream scatterStream = createDeferred(backingStoreSupplier);
    086                streams.add(scatterStream);
    087                return scatterStream;
    088            } catch (final IOException e) {
    089                throw new RuntimeException(e);
    090            }
    091        }
    092    };
    093
    094    /**
    095     * Create a ParallelScatterZipCreator with default threads, which is set to the number of available
    096     * processors, as defined by {@link java.lang.Runtime#availableProcessors}
    097     */
    098    public ParallelScatterZipCreator() {
    099        this(Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()));
    100    }
    101
    102    /**
    103     * Create a ParallelScatterZipCreator
    104     *
    105     * @param executorService The executorService to use for parallel scheduling. For technical reasons,
    106     *                        this will be shut down by this class.
    107     */
    108    public ParallelScatterZipCreator(final ExecutorService executorService) {
    109        this(executorService, new DefaultBackingStoreSupplier());
    110    }
    111
    112    /**
    113     * Create a ParallelScatterZipCreator
    114     *
    115     * @param executorService The executorService to use. For technical reasons, this will be shut down
    116     *                        by this class.
    117     * @param backingStoreSupplier The supplier of backing store which shall be used
    118     */
    119    public ParallelScatterZipCreator(final ExecutorService executorService,
    120                                     final ScatterGatherBackingStoreSupplier backingStoreSupplier) {
    121        this.backingStoreSupplier = backingStoreSupplier;
    122        es = executorService;
    123    }
    124
    125    /**
    126     * Adds an archive entry to this archive.
    127     * <p>
    128     * This method is expected to be called from a single client thread
    129     * </p>
    130     *
    131     * @param zipArchiveEntry The entry to add.
    132     * @param source          The source input stream supplier
    133     */
    134
    135    public void addArchiveEntry(final ZipArchiveEntry zipArchiveEntry, final InputStreamSupplier source) {
    136        submit(createCallable(zipArchiveEntry, source));
    137    }
    138
    139    /**
    140     * Submit a callable for compression.
    141     *
    142     * @see ParallelScatterZipCreator#createCallable for details of if/when to use this.
    143     *
    144     * @param callable The callable to run, created by {@link #createCallable createCallable}, possibly wrapped by caller.
    145     */
    146    public final void submit(final Callable<Object> callable) {
    147        futures.add(es.submit(callable));
    148    }
    149
    150    /**
    151     * Create a callable that will compress the given archive entry.
    152     *
    153     * <p>This method is expected to be called from a single client thread.</p>
    154     *
    155     * Consider using {@link #addArchiveEntry addArchiveEntry}, which wraps this method and {@link #submit submit}.
    156     * The most common use case for using {@link #createCallable createCallable} and {@link #submit submit} from a
    157     * client is if you want to wrap the callable in something that can be prioritized by the supplied
    158     * {@link ExecutorService}, for instance to process large or slow files first.
    159     * Since the creation of the {@link ExecutorService} is handled by the client, all of this is up to the client.
    160     *
    161     * @param zipArchiveEntry The entry to add.
    162     * @param source          The source input stream supplier
    163     * @return A callable that should subsequently passed to #submit, possibly in a wrapped/adapted from. The
    164     * value of this callable is not used, but any exceptions happening inside the compression
    165     * will be propagated through the callable.
    166     */
    167
    168    public final Callable<Object> createCallable(final ZipArchiveEntry zipArchiveEntry, final InputStreamSupplier source) {
    169        final int method = zipArchiveEntry.getMethod();
    170        if (method == ZipMethod.UNKNOWN_CODE) {
    171            throw new IllegalArgumentException("Method must be set on zipArchiveEntry: " + zipArchiveEntry);
    172        }
    173        final ZipArchiveEntryRequest zipArchiveEntryRequest = createZipArchiveEntryRequest(zipArchiveEntry, source);
    174        return new Callable<Object>() {
    175            @Override
    176            public Object call() throws Exception {
    177                tlScatterStreams.get().addArchiveEntry(zipArchiveEntryRequest);
    178                return null;
    179            }
    180        };
    181    }
    182
    183
    184    /**
    185     * Write the contents this to the target {@link ZipArchiveOutputStream}.
    186     * <p>
    187     * It may be beneficial to write things like directories and manifest files to the targetStream
    188     * before calling this method.
    189     * </p>
    190     *
    191     * @param targetStream The {@link ZipArchiveOutputStream} to receive the contents of the scatter streams
    192     * @throws IOException          If writing fails
    193     * @throws InterruptedException If we get interrupted
    194     * @throws ExecutionException   If something happens in the parallel execution
    195     */
    196    public void writeTo(final ZipArchiveOutputStream targetStream)
    197            throws IOException, InterruptedException, ExecutionException {
    198
    199        // Make sure we catch any exceptions from parallel phase
    200        for (final Future<?> future : futures) {
    201            future.get();
    202        }
    203
    204        es.shutdown();
    205        es.awaitTermination(1000 * 60, TimeUnit.SECONDS);  // == Infinity. We really *must* wait for this to complete
    206
    207        // It is important that all threads terminate before we go on, ensure happens-before relationship
    208        compressionDoneAt = System.currentTimeMillis();
    209
    210        for (final ScatterZipOutputStream scatterStream : streams) {
    211            scatterStream.writeTo(targetStream);
    212            scatterStream.close();
    213        }
    214
    215        scatterDoneAt = System.currentTimeMillis();
    216    }
    217
    218    /**
    219     * Returns a message describing the overall statistics of the compression run
    220     *
    221     * @return A string
    222     */
    223    public ScatterStatistics getStatisticsMessage() {
    224        return new ScatterStatistics(compressionDoneAt - startedAt, scatterDoneAt - compressionDoneAt);
    225    }
    226}
    227