Skip to content

Commit 3e55044

Browse files
maskaravivekashishkumar468
maskaravivek
authored and
ashishkumar468
committed
With chunked uploads (commons-app#3855)
1 parent c11083a commit 3e55044

File tree

3 files changed

+103
-46
lines changed

3 files changed

+103
-46
lines changed

‎app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java

+101-44
Original file line numberDiff line numberDiff line change
@@ -4,71 +4,128 @@
44

55
import android.content.Context;
66
import android.net.Uri;
7+
import androidx.annotation.Nullable;
78
import fr.free.nrw.commons.CommonsApplication;
89
import fr.free.nrw.commons.contributions.Contribution;
910
import fr.free.nrw.commons.upload.UploadService.NotificationUpdateProgressListener;
1011
import io.reactivex.Observable;
1112
import java.io.File;
13+
import java.io.FileOutputStream;
14+
import java.io.IOException;
15+
import java.util.concurrent.atomic.AtomicReference;
1216
import javax.inject.Inject;
1317
import javax.inject.Named;
1418
import javax.inject.Singleton;
1519
import okhttp3.MediaType;
1620
import okhttp3.MultipartBody;
1721
import okhttp3.RequestBody;
1822
import org.wikipedia.csrf.CsrfTokenClient;
23+
import timber.log.Timber;
1924

2025
@Singleton
2126
public class UploadClient {
2227

23-
private final UploadInterface uploadInterface;
24-
private final CsrfTokenClient csrfTokenClient;
25-
private final PageContentsCreator pageContentsCreator;
28+
private final int CHUNK_SIZE = 256 * 1024; // 256 KB
2629

27-
@Inject
28-
public UploadClient(UploadInterface uploadInterface,
29-
@Named(NAMED_COMMONS_CSRF) CsrfTokenClient csrfTokenClient,
30-
PageContentsCreator pageContentsCreator) {
31-
this.uploadInterface = uploadInterface;
32-
this.csrfTokenClient = csrfTokenClient;
33-
this.pageContentsCreator = pageContentsCreator;
34-
}
30+
private final UploadInterface uploadInterface;
31+
private final CsrfTokenClient csrfTokenClient;
32+
private final PageContentsCreator pageContentsCreator;
33+
private final FileUtilsWrapper fileUtilsWrapper;
34+
35+
@Inject
36+
public UploadClient(final UploadInterface uploadInterface,
37+
@Named(NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient,
38+
final PageContentsCreator pageContentsCreator,
39+
final FileUtilsWrapper fileUtilsWrapper) {
40+
this.uploadInterface = uploadInterface;
41+
this.csrfTokenClient = csrfTokenClient;
42+
this.pageContentsCreator = pageContentsCreator;
43+
this.fileUtilsWrapper = fileUtilsWrapper;
44+
}
3545

36-
Observable<UploadResult> uploadFileToStash(Context context, String filename, File file,
37-
NotificationUpdateProgressListener notificationUpdater) {
38-
RequestBody requestBody = RequestBody
39-
.create(MediaType.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath()))), file);
46+
/**
47+
* Upload file to stash in chunks of specified size. Uploading files in chunks will make handling
48+
* of large files easier. Also, it will be useful in supporting pause/resume of uploads
49+
*/
50+
Observable<UploadResult> uploadFileToStash(
51+
final Context context, final String filename, final File file,
52+
final NotificationUpdateProgressListener notificationUpdater) throws IOException {
53+
final Observable<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
54+
final MediaType mediaType = MediaType
55+
.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath())));
4056

41-
CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
42-
(bytesWritten, contentLength) -> notificationUpdater
43-
.onProgress(bytesWritten, contentLength));
57+
final long[] offset = {0};
58+
final String[] fileKey = {null};
59+
final AtomicReference<UploadResult> result = new AtomicReference<>();
60+
fileChunks.blockingForEach(chunkFile -> {
61+
final RequestBody requestBody = RequestBody
62+
.create(mediaType, chunkFile);
63+
final CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
64+
notificationUpdater::onProgress, offset[0], file.length());
65+
uploadChunkToStash(filename,
66+
file.length(),
67+
offset[0],
68+
fileKey[0],
69+
countingRequestBody).blockingSubscribe(uploadResult -> {
70+
result.set(uploadResult);
71+
offset[0] = uploadResult.getOffset();
72+
fileKey[0] = uploadResult.getFilekey();
73+
});
74+
});
75+
return Observable.just(result.get());
76+
}
4477

45-
MultipartBody.Part filePart = MultipartBody.Part.createFormData("file", filename, countingRequestBody);
46-
RequestBody fileNameRequestBody = RequestBody.create(okhttp3.MultipartBody.FORM, filename);
47-
RequestBody tokenRequestBody;
48-
try {
49-
tokenRequestBody = RequestBody.create(MultipartBody.FORM, csrfTokenClient.getTokenBlocking());
50-
return uploadInterface.uploadFileToStash(fileNameRequestBody, tokenRequestBody, filePart)
51-
.map(stashUploadResponse -> stashUploadResponse.getUpload());
52-
} catch (Throwable throwable) {
53-
throwable.printStackTrace();
54-
return Observable.error(throwable);
55-
}
78+
/**
79+
* Uploads a file chunk to stash
80+
*
81+
* @param filename The name of the file being uploaded
82+
* @param fileSize The total size of the file
83+
* @param offset The offset returned by the previous chunk upload
84+
* @param fileKey The filekey returned by the previous chunk upload
85+
* @param countingRequestBody Request body with chunk file
86+
* @return
87+
*/
88+
Observable<UploadResult> uploadChunkToStash(final String filename,
89+
final long fileSize,
90+
final long offset,
91+
final String fileKey,
92+
final CountingRequestBody countingRequestBody) {
93+
final MultipartBody.Part filePart = MultipartBody.Part
94+
.createFormData("chunk", filename, countingRequestBody);
95+
try {
96+
return uploadInterface.uploadFileToStash(toRequestBody(filename),
97+
toRequestBody(String.valueOf(fileSize)),
98+
toRequestBody(String.valueOf(offset)),
99+
toRequestBody(fileKey),
100+
toRequestBody(csrfTokenClient.getTokenBlocking()),
101+
filePart)
102+
.map(UploadResponse::getUpload);
103+
} catch (final Throwable throwable) {
104+
Timber.e(throwable, "Failed to upload chunk to stash");
105+
return Observable.error(throwable);
56106
}
107+
}
108+
109+
@Nullable
110+
private RequestBody toRequestBody(@Nullable final String value) {
111+
return value == null ? null : RequestBody.create(okhttp3.MultipartBody.FORM, value);
112+
}
113+
57114

58-
Observable<UploadResult> uploadFileFromStash(Context context,
59-
Contribution contribution,
60-
String uniqueFileName,
61-
String fileKey) {
62-
try {
63-
return uploadInterface
64-
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
65-
pageContentsCreator.createFrom(contribution),
66-
CommonsApplication.DEFAULT_EDIT_SUMMARY,
67-
uniqueFileName,
68-
fileKey).map(uploadResponse -> uploadResponse.getUpload());
69-
} catch (Throwable throwable) {
70-
throwable.printStackTrace();
71-
return Observable.error(throwable);
72-
}
115+
Observable<UploadResult> uploadFileFromStash(final Context context,
116+
final Contribution contribution,
117+
final String uniqueFileName,
118+
final String fileKey) {
119+
try {
120+
return uploadInterface
121+
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
122+
pageContentsCreator.createFrom(contribution),
123+
CommonsApplication.DEFAULT_EDIT_SUMMARY,
124+
uniqueFileName,
125+
fileKey).map(UploadResponse::getUpload);
126+
} catch (final Throwable throwable) {
127+
throwable.printStackTrace();
128+
return Observable.error(throwable);
73129
}
130+
}
74131
}

‎app/src/main/java/fr/free/nrw/commons/upload/UploadInterface.java

+1-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import androidx.annotation.NonNull;
44

5-
import com.google.gson.JsonObject;
65
import io.reactivex.Observable;
76
import okhttp3.MultipartBody;
87
import okhttp3.RequestBody;
@@ -30,7 +29,7 @@ Observable<UploadResponse> uploadFileToStash(@Part("filename") RequestBody filen
3029
@POST(MW_API_PREFIX + "action=upload&ignorewarnings=1")
3130
@FormUrlEncoded
3231
@NonNull
33-
Observable<JsonObject> uploadFileFromStash(@NonNull @Field("token") String token,
32+
Observable<UploadResponse> uploadFileFromStash(@NonNull @Field("token") String token,
3433
@NonNull @Field("text") String text,
3534
@NonNull @Field("comment") String comment,
3635
@NonNull @Field("filename") String filename,

‎app/src/main/java/fr/free/nrw/commons/upload/UploadResult.kt

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ private const val RESULT_SUCCESS = "Success"
77
data class UploadResult(
88
val result: String,
99
val filekey: String,
10+
val offset: Int,
1011
val filename: String,
1112
val sessionkey: String,
1213
val imageinfo: ImageInfo

0 commit comments

Comments
 (0)