Skip to content

Commit 3361155

Browse files
maskaravivekmaskaravivek
authored
With chunked uploads (commons-app#3855)
1 parent f26784e commit 3361155

File tree

5 files changed

+191
-73
lines changed

5 files changed

+191
-73
lines changed

‎app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt

+8-2
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,12 @@ import java.io.IOException
1212
*
1313
* @author Ashish Kumar
1414
*/
15-
class CountingRequestBody(protected var delegate: RequestBody, protected var listener: Listener) : RequestBody() {
15+
class CountingRequestBody(
16+
protected var delegate: RequestBody,
17+
protected var listener: Listener,
18+
var offset: Long,
19+
var totalContentLength: Long
20+
) : RequestBody() {
1621
protected var countingSink: CountingSink? = null
1722
override fun contentType(): MediaType? {
1823
return delegate.contentType()
@@ -37,11 +42,12 @@ class CountingRequestBody(protected var delegate: RequestBody, protected var lis
3742

3843
protected inner class CountingSink(delegate: Sink?) : ForwardingSink(delegate!!) {
3944
private var bytesWritten: Long = 0
45+
4046
@Throws(IOException::class)
4147
override fun write(source: Buffer, byteCount: Long) {
4248
super.write(source, byteCount)
4349
bytesWritten += byteCount
44-
listener.onRequestProgress(bytesWritten, contentLength())
50+
listener.onRequestProgress(offset + bytesWritten, totalContentLength)
4551
}
4652
}
4753

‎app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java

+64-13
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,84 @@
11
package fr.free.nrw.commons.upload;
22

3+
import android.content.Context;
4+
import io.reactivex.Observable;
5+
import java.io.BufferedInputStream;
6+
import java.io.File;
37
import java.io.FileInputStream;
48
import java.io.FileNotFoundException;
9+
import java.io.FileOutputStream;
10+
import java.io.IOException;
511
import java.io.InputStream;
612

13+
import java.util.ArrayList;
14+
import java.util.Arrays;
15+
import java.util.List;
716
import javax.inject.Inject;
817
import javax.inject.Singleton;
18+
import timber.log.Timber;
919

1020
@Singleton
1121
public class FileUtilsWrapper {
1222

13-
@Inject
14-
public FileUtilsWrapper() {
23+
@Inject
24+
public FileUtilsWrapper() {
1525

16-
}
26+
}
1727

18-
public String getFileExt(String fileName) {
19-
return FileUtils.getFileExt(fileName);
20-
}
28+
public String getFileExt(String fileName) {
29+
return FileUtils.getFileExt(fileName);
30+
}
2131

22-
public String getSHA1(InputStream is) {
23-
return FileUtils.getSHA1(is);
24-
}
32+
public String getSHA1(InputStream is) {
33+
return FileUtils.getSHA1(is);
34+
}
35+
36+
public FileInputStream getFileInputStream(String filePath) throws FileNotFoundException {
37+
return FileUtils.getFileInputStream(filePath);
38+
}
39+
40+
public String getGeolocationOfFile(String filePath) {
41+
return FileUtils.getGeolocationOfFile(filePath);
42+
}
43+
44+
45+
/**
46+
* Takes a file as input and returns an Observable of files with the specified chunk size
47+
*/
48+
public Observable<File> getFileChunks(Context context, File file, final int chunkSize)
49+
throws IOException {
50+
final byte[] buffer = new byte[chunkSize];
2551

26-
public FileInputStream getFileInputStream(String filePath) throws FileNotFoundException {
27-
return FileUtils.getFileInputStream(filePath);
52+
//try-with-resources to ensure closing stream
53+
try (final FileInputStream fis = new FileInputStream(file);
54+
final BufferedInputStream bis = new BufferedInputStream(fis)) {
55+
final List<File> buffers = new ArrayList<>();
56+
int size;
57+
while ((size = bis.read(buffer)) > 0) {
58+
buffers.add(writeToFile(context, Arrays.copyOf(buffer, size), file.getName(),
59+
getFileExt(file.getName())));
60+
}
61+
return Observable.fromIterable(buffers);
2862
}
63+
}
2964

30-
public String getGeolocationOfFile(String filePath) {
31-
return FileUtils.getGeolocationOfFile(filePath);
65+
/**
66+
* Create a temp file containing the passed byte data.
67+
*/
68+
private File writeToFile(Context context, final byte[] data, final String fileName,
69+
String fileExtension)
70+
throws IOException {
71+
final File file = File.createTempFile(fileName, fileExtension, context.getCacheDir());
72+
try {
73+
if (!file.exists()) {
74+
file.createNewFile();
75+
}
76+
final FileOutputStream fos = new FileOutputStream(file);
77+
fos.write(data);
78+
fos.close();
79+
} catch (final Exception throwable) {
80+
Timber.e(throwable, "Failed to create file");
3281
}
82+
return file;
83+
}
3384
}

‎app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java

+101-44
Original file line numberDiff line numberDiff line change
@@ -4,71 +4,128 @@
44

55
import android.content.Context;
66
import android.net.Uri;
7+
import androidx.annotation.Nullable;
78
import fr.free.nrw.commons.CommonsApplication;
89
import fr.free.nrw.commons.contributions.Contribution;
910
import fr.free.nrw.commons.upload.UploadService.NotificationUpdateProgressListener;
1011
import io.reactivex.Observable;
1112
import java.io.File;
13+
import java.io.FileOutputStream;
14+
import java.io.IOException;
15+
import java.util.concurrent.atomic.AtomicReference;
1216
import javax.inject.Inject;
1317
import javax.inject.Named;
1418
import javax.inject.Singleton;
1519
import okhttp3.MediaType;
1620
import okhttp3.MultipartBody;
1721
import okhttp3.RequestBody;
1822
import org.wikipedia.csrf.CsrfTokenClient;
23+
import timber.log.Timber;
1924

2025
@Singleton
2126
public class UploadClient {
2227

23-
private final UploadInterface uploadInterface;
24-
private final CsrfTokenClient csrfTokenClient;
25-
private final PageContentsCreator pageContentsCreator;
28+
private final int CHUNK_SIZE = 256 * 1024; // 256 KB
2629

27-
@Inject
28-
public UploadClient(UploadInterface uploadInterface,
29-
@Named(NAMED_COMMONS_CSRF) CsrfTokenClient csrfTokenClient,
30-
PageContentsCreator pageContentsCreator) {
31-
this.uploadInterface = uploadInterface;
32-
this.csrfTokenClient = csrfTokenClient;
33-
this.pageContentsCreator = pageContentsCreator;
34-
}
30+
private final UploadInterface uploadInterface;
31+
private final CsrfTokenClient csrfTokenClient;
32+
private final PageContentsCreator pageContentsCreator;
33+
private final FileUtilsWrapper fileUtilsWrapper;
34+
35+
@Inject
36+
public UploadClient(final UploadInterface uploadInterface,
37+
@Named(NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient,
38+
final PageContentsCreator pageContentsCreator,
39+
final FileUtilsWrapper fileUtilsWrapper) {
40+
this.uploadInterface = uploadInterface;
41+
this.csrfTokenClient = csrfTokenClient;
42+
this.pageContentsCreator = pageContentsCreator;
43+
this.fileUtilsWrapper = fileUtilsWrapper;
44+
}
3545

36-
Observable<UploadResult> uploadFileToStash(Context context, String filename, File file,
37-
NotificationUpdateProgressListener notificationUpdater) {
38-
RequestBody requestBody = RequestBody
39-
.create(MediaType.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath()))), file);
46+
/**
47+
* Upload file to stash in chunks of specified size. Uploading files in chunks will make handling
48+
* of large files easier. Also, it will be useful in supporting pause/resume of uploads
49+
*/
50+
Observable<UploadResult> uploadFileToStash(
51+
final Context context, final String filename, final File file,
52+
final NotificationUpdateProgressListener notificationUpdater) throws IOException {
53+
final Observable<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
54+
final MediaType mediaType = MediaType
55+
.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath())));
4056

41-
CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
42-
(bytesWritten, contentLength) -> notificationUpdater
43-
.onProgress(bytesWritten, contentLength));
57+
final long[] offset = {0};
58+
final String[] fileKey = {null};
59+
final AtomicReference<UploadResult> result = new AtomicReference<>();
60+
fileChunks.blockingForEach(chunkFile -> {
61+
final RequestBody requestBody = RequestBody
62+
.create(mediaType, chunkFile);
63+
final CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
64+
notificationUpdater::onProgress, offset[0], file.length());
65+
uploadChunkToStash(filename,
66+
file.length(),
67+
offset[0],
68+
fileKey[0],
69+
countingRequestBody).blockingSubscribe(uploadResult -> {
70+
result.set(uploadResult);
71+
offset[0] = uploadResult.getOffset();
72+
fileKey[0] = uploadResult.getFilekey();
73+
});
74+
});
75+
return Observable.just(result.get());
76+
}
4477

45-
MultipartBody.Part filePart = MultipartBody.Part.createFormData("file", filename, countingRequestBody);
46-
RequestBody fileNameRequestBody = RequestBody.create(okhttp3.MultipartBody.FORM, filename);
47-
RequestBody tokenRequestBody;
48-
try {
49-
tokenRequestBody = RequestBody.create(MultipartBody.FORM, csrfTokenClient.getTokenBlocking());
50-
return uploadInterface.uploadFileToStash(fileNameRequestBody, tokenRequestBody, filePart)
51-
.map(stashUploadResponse -> stashUploadResponse.getUpload());
52-
} catch (Throwable throwable) {
53-
throwable.printStackTrace();
54-
return Observable.error(throwable);
55-
}
78+
/**
79+
* Uploads a file chunk to stash
80+
*
81+
* @param filename The name of the file being uploaded
82+
* @param fileSize The total size of the file
83+
* @param offset The offset returned by the previous chunk upload
84+
* @param fileKey The filekey returned by the previous chunk upload
85+
* @param countingRequestBody Request body with chunk file
86+
* @return
87+
*/
88+
Observable<UploadResult> uploadChunkToStash(final String filename,
89+
final long fileSize,
90+
final long offset,
91+
final String fileKey,
92+
final CountingRequestBody countingRequestBody) {
93+
final MultipartBody.Part filePart = MultipartBody.Part
94+
.createFormData("chunk", filename, countingRequestBody);
95+
try {
96+
return uploadInterface.uploadFileToStash(toRequestBody(filename),
97+
toRequestBody(String.valueOf(fileSize)),
98+
toRequestBody(String.valueOf(offset)),
99+
toRequestBody(fileKey),
100+
toRequestBody(csrfTokenClient.getTokenBlocking()),
101+
filePart)
102+
.map(UploadResponse::getUpload);
103+
} catch (final Throwable throwable) {
104+
Timber.e(throwable, "Failed to upload chunk to stash");
105+
return Observable.error(throwable);
56106
}
107+
}
108+
109+
@Nullable
110+
private RequestBody toRequestBody(@Nullable final String value) {
111+
return value == null ? null : RequestBody.create(okhttp3.MultipartBody.FORM, value);
112+
}
113+
57114

58-
Observable<UploadResult> uploadFileFromStash(Context context,
59-
Contribution contribution,
60-
String uniqueFileName,
61-
String fileKey) {
62-
try {
63-
return uploadInterface
64-
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
65-
pageContentsCreator.createFrom(contribution),
66-
CommonsApplication.DEFAULT_EDIT_SUMMARY,
67-
uniqueFileName,
68-
fileKey).map(uploadResponse -> uploadResponse.getUpload());
69-
} catch (Throwable throwable) {
70-
throwable.printStackTrace();
71-
return Observable.error(throwable);
72-
}
115+
Observable<UploadResult> uploadFileFromStash(final Context context,
116+
final Contribution contribution,
117+
final String uniqueFileName,
118+
final String fileKey) {
119+
try {
120+
return uploadInterface
121+
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
122+
pageContentsCreator.createFrom(contribution),
123+
CommonsApplication.DEFAULT_EDIT_SUMMARY,
124+
uniqueFileName,
125+
fileKey).map(UploadResponse::getUpload);
126+
} catch (final Throwable throwable) {
127+
throwable.printStackTrace();
128+
return Observable.error(throwable);
73129
}
130+
}
74131
}

‎app/src/main/java/fr/free/nrw/commons/upload/UploadInterface.java

+17-14
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,22 @@
1616

1717
public interface UploadInterface {
1818

19-
@Multipart
20-
@POST(MW_API_PREFIX + "action=upload&stash=1&ignorewarnings=1")
21-
Observable<UploadResponse> uploadFileToStash(@Part("filename") RequestBody filename,
22-
@Part("token") RequestBody token,
23-
@Part MultipartBody.Part filePart);
19+
@Multipart
20+
@POST(MW_API_PREFIX + "action=upload&stash=1&ignorewarnings=1")
21+
Observable<UploadResponse> uploadFileToStash(@Part("filename") RequestBody filename,
22+
@Part("filesize") RequestBody totalFileSize,
23+
@Part("offset") RequestBody offset,
24+
@Part("filekey") RequestBody fileKey,
25+
@Part("token") RequestBody token,
26+
@Part MultipartBody.Part filePart);
2427

25-
@Headers("Cache-Control: no-cache")
26-
@POST(MW_API_PREFIX + "action=upload&ignorewarnings=1")
27-
@FormUrlEncoded
28-
@NonNull
29-
Observable<UploadResponse> uploadFileFromStash(@NonNull @Field("token") String token,
30-
@NonNull @Field("text") String text,
31-
@NonNull @Field("comment") String comment,
32-
@NonNull @Field("filename") String filename,
33-
@NonNull @Field("filekey") String filekey);
28+
@Headers("Cache-Control: no-cache")
29+
@POST(MW_API_PREFIX + "action=upload&ignorewarnings=1")
30+
@FormUrlEncoded
31+
@NonNull
32+
Observable<UploadResponse> uploadFileFromStash(@NonNull @Field("token") String token,
33+
@NonNull @Field("text") String text,
34+
@NonNull @Field("comment") String comment,
35+
@NonNull @Field("filename") String filename,
36+
@NonNull @Field("filekey") String filekey);
3437
}

‎app/src/main/java/fr/free/nrw/commons/upload/UploadResult.kt

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ private const val RESULT_SUCCESS = "Success"
77
data class UploadResult(
88
val result: String,
99
val filekey: String,
10+
val offset: Int,
1011
val filename: String,
1112
val sessionkey: String,
1213
val imageinfo: ImageInfo

0 commit comments

Comments
 (0)