Skip to content

Commit 6c55525

Browse files
maskaravivekmaskaravivek
authored
Handle failures in chunk uploads (commons-app#3916)
* Handle failures in chunk uploads * Fix failures * Upload fixed * Handle multiple file upload * Increase request timeout
1 parent 0d5fa04 commit 6c55525

File tree

8 files changed

+69
-38
lines changed

8 files changed

+69
-38
lines changed

‎app/src/main/java/fr/free/nrw/commons/OkHttpConnectionFactory.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import java.util.Arrays;
77
import java.util.Collections;
88
import java.util.List;
9+
import java.util.concurrent.TimeUnit;
910
import okhttp3.Cache;
1011
import okhttp3.Interceptor;
1112
import okhttp3.OkHttpClient;
@@ -36,6 +37,9 @@ private static OkHttpClient createClient() {
3637
return new OkHttpClient.Builder()
3738
.cookieJar(SharedPreferenceCookieManager.getInstance())
3839
.cache(NET_CACHE)
40+
.connectTimeout(60, TimeUnit.SECONDS)
41+
.writeTimeout(60, TimeUnit.SECONDS)
42+
.readTimeout(60, TimeUnit.SECONDS)
3943
.addInterceptor(getLoggingInterceptor())
4044
.addInterceptor(new UnsuccessfulResponseInterceptor())
4145
.addInterceptor(new CommonHeaderRequestInterceptor())

‎app/src/main/java/fr/free/nrw/commons/contributions/ChunkInfo.kt

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,20 +6,20 @@ import fr.free.nrw.commons.upload.UploadResult
66

77
data class ChunkInfo(
88
val uploadResult: UploadResult,
9-
val lastChunkIndex: Int,
10-
var isLastChunkUploaded: Boolean
9+
val indexOfNextChunkToUpload: Int,
10+
val totalChunks: Int
1111
) : Parcelable {
1212
constructor(parcel: Parcel) : this(
1313
parcel.readParcelable(UploadResult::class.java.classLoader),
1414
parcel.readInt(),
15-
parcel.readByte() != 0.toByte()
15+
parcel.readInt()
1616
) {
1717
}
1818

1919
override fun writeToParcel(parcel: Parcel, flags: Int) {
2020
parcel.writeParcelable(uploadResult, flags)
21-
parcel.writeInt(lastChunkIndex)
22-
parcel.writeByte(if (isLastChunkUploaded) 1 else 0)
21+
parcel.writeInt(indexOfNextChunkToUpload)
22+
parcel.writeInt(totalChunks)
2323
}
2424

2525
override fun describeContents(): Int {

‎app/src/main/java/fr/free/nrw/commons/db/AppDatabase.kt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import fr.free.nrw.commons.contributions.ContributionDao
1010
* The database for accessing the respective DAOs
1111
*
1212
*/
13-
@Database(entities = [Contribution::class], version = 5, exportSchema = false)
13+
@Database(entities = [Contribution::class], version = 6, exportSchema = false)
1414
@TypeConverters(Converters::class)
1515
abstract class AppDatabase : RoomDatabase() {
1616
abstract fun contributionDao(): ContributionDao

‎app/src/main/java/fr/free/nrw/commons/di/NetworkingModule.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,8 @@ public class NetworkingModule {
6262
public OkHttpClient provideOkHttpClient(Context context,
6363
HttpLoggingInterceptor httpLoggingInterceptor) {
6464
File dir = new File(context.getCacheDir(), "okHttpCache");
65-
return new OkHttpClient.Builder().connectTimeout(60, TimeUnit.SECONDS)
65+
return new OkHttpClient.Builder()
66+
.connectTimeout(60, TimeUnit.SECONDS)
6667
.writeTimeout(60, TimeUnit.SECONDS)
6768
.addInterceptor(httpLoggingInterceptor)
6869
.readTimeout(60, TimeUnit.SECONDS)

‎app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ public String getGeolocationOfFile(String filePath) {
4545
/**
4646
* Takes a file as input and returns an Observable of files with the specified chunk size
4747
*/
48-
public Observable<File> getFileChunks(Context context, File file, final int chunkSize)
48+
public List<File> getFileChunks(Context context, File file, final int chunkSize)
4949
throws IOException {
5050
final byte[] buffer = new byte[chunkSize];
5151

@@ -58,7 +58,7 @@ public Observable<File> getFileChunks(Context context, File file, final int chun
5858
buffers.add(writeToFile(context, Arrays.copyOf(buffer, size), file.getName(),
5959
getFileExt(file.getName())));
6060
}
61-
return Observable.fromIterable(buffers);
61+
return buffers;
6262
}
6363
}
6464

‎app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java

Lines changed: 50 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,10 @@
1515
import java.io.File;
1616
import java.io.IOException;
1717
import java.util.Date;
18+
import java.util.HashMap;
19+
import java.util.List;
20+
import java.util.Map;
21+
import java.util.concurrent.atomic.AtomicBoolean;
1822
import java.util.concurrent.atomic.AtomicInteger;
1923
import java.util.concurrent.atomic.AtomicReference;
2024
import javax.inject.Inject;
@@ -41,7 +45,8 @@ public class UploadClient {
4145
private final PageContentsCreator pageContentsCreator;
4246
private final FileUtilsWrapper fileUtilsWrapper;
4347
private final Gson gson;
44-
private boolean pauseUploads = false;
48+
49+
private Map<String, Boolean> pauseUploads;
4550

4651
private final CompositeDisposable compositeDisposable = new CompositeDisposable();
4752

@@ -55,6 +60,7 @@ public UploadClient(final UploadInterface uploadInterface,
5560
this.pageContentsCreator = pageContentsCreator;
5661
this.fileUtilsWrapper = fileUtilsWrapper;
5762
this.gson = gson;
63+
this.pauseUploads = new HashMap<>();
5864
}
5965

6066
/**
@@ -64,32 +70,50 @@ public UploadClient(final UploadInterface uploadInterface,
6470
Observable<StashUploadResult> uploadFileToStash(
6571
final Context context, final String filename, final Contribution contribution,
6672
final NotificationUpdateProgressListener notificationUpdater) throws IOException {
67-
if (contribution.getChunkInfo() != null && contribution.getChunkInfo().isLastChunkUploaded()) {
73+
if (contribution.getChunkInfo() != null
74+
&& contribution.getChunkInfo().getTotalChunks() == contribution.getChunkInfo()
75+
.getIndexOfNextChunkToUpload()) {
6876
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
6977
contribution.getChunkInfo().getUploadResult().getFilekey()));
7078
}
71-
pauseUploads = false;
72-
File file = new File(contribution.getLocalUri().getPath());
73-
final Observable<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
79+
80+
pauseUploads.put(contribution.getPageId(), false);
81+
82+
final File file = new File(contribution.getLocalUri().getPath());
83+
final List<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
84+
85+
final int totalChunks = fileChunks.size();
86+
7487
final MediaType mediaType = MediaType
7588
.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath())));
7689

77-
final AtomicInteger index = new AtomicInteger();
7890
final AtomicReference<ChunkInfo> chunkInfo = new AtomicReference<>();
79-
Timber.d("Chunk info");
80-
if (contribution.getChunkInfo() != null && isStashValid(contribution)) {
91+
if (isStashValid(contribution)) {
8192
chunkInfo.set(contribution.getChunkInfo());
93+
94+
Timber.d("Chunk: Next Chunk: %s, Total Chunks: %s",
95+
contribution.getChunkInfo().getIndexOfNextChunkToUpload(),
96+
contribution.getChunkInfo().getTotalChunks());
8297
}
83-
compositeDisposable.add(fileChunks.forEach(chunkFile -> {
84-
if (pauseUploads) {
98+
99+
final AtomicInteger index = new AtomicInteger();
100+
final AtomicBoolean failures = new AtomicBoolean();
101+
102+
compositeDisposable.add(Observable.fromIterable(fileChunks).forEach(chunkFile -> {
103+
if (pauseUploads.get(contribution.getPageId()) || failures.get()) {
85104
return;
86105
}
87-
if (chunkInfo.get() != null && index.get() < chunkInfo.get().getLastChunkIndex()) {
88-
index.getAndIncrement();
106+
107+
if (chunkInfo.get() != null && index.get() < chunkInfo.get().getIndexOfNextChunkToUpload()) {
108+
index.incrementAndGet();
109+
Timber.d("Chunk: Increment and return: %s", index.get());
89110
return;
90111
}
112+
index.getAndIncrement();
91113
final int offset =
92114
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getOffset() : 0;
115+
116+
Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset);
93117
final String filekey =
94118
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getFilekey() : null;
95119

@@ -104,17 +128,20 @@ Observable<StashUploadResult> uploadFileToStash(
104128
offset,
105129
filekey,
106130
countingRequestBody).subscribe(uploadResult -> {
107-
chunkInfo.set(new ChunkInfo(uploadResult, index.incrementAndGet(), false));
131+
Timber.d("Chunk: Received Chunk number: %s, offset: %s", index.get(),
132+
uploadResult.getOffset());
133+
chunkInfo.set(
134+
new ChunkInfo(uploadResult, index.get(), totalChunks));
108135
notificationUpdater.onChunkUploaded(contribution, chunkInfo.get());
109136
}, throwable -> {
110-
Timber.e(throwable, "Error occurred in uploading chunk");
137+
failures.set(true);
111138
}));
112139
}));
113140

114-
chunkInfo.get().setLastChunkUploaded(true);
115-
notificationUpdater.onChunkUploaded(contribution, chunkInfo.get());
116-
if (pauseUploads) {
141+
if (pauseUploads.get(contribution.getPageId())) {
117142
return Observable.just(new StashUploadResult(StashUploadState.PAUSED, null));
143+
} else if (failures.get()) {
144+
return Observable.just(new StashUploadResult(StashUploadState.FAILED, null));
118145
} else if (chunkInfo.get() != null) {
119146
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
120147
chunkInfo.get().getUploadResult().getFilekey()));
@@ -129,8 +156,9 @@ Observable<StashUploadResult> uploadFileToStash(
129156
* @return
130157
*/
131158
private boolean isStashValid(Contribution contribution) {
132-
return contribution.getDateModified()
133-
.after(new Date(System.currentTimeMillis() - MAX_CHUNK_AGE));
159+
return contribution.getChunkInfo() != null &&
160+
contribution.getDateModified()
161+
.after(new Date(System.currentTimeMillis() - MAX_CHUNK_AGE));
134162
}
135163

136164
/**
@@ -166,9 +194,10 @@ Observable<UploadResult> uploadChunkToStash(final String filename,
166194

167195
/**
168196
* Dispose the active disposable and sets the pause variable
197+
* @param pageId
169198
*/
170-
public void pauseUpload() {
171-
pauseUploads = true;
199+
public void pauseUpload(String pageId) {
200+
pauseUploads.put(pageId, true);
172201
if (!compositeDisposable.isDisposed()) {
173202
compositeDisposable.dispose();
174203
}

‎app/src/main/java/fr/free/nrw/commons/upload/UploadService.java

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ public void onChunkUploaded(Contribution contribution, ChunkInfo chunkInfo) {
148148
* @param contribution
149149
*/
150150
public void pauseUpload(Contribution contribution) {
151-
uploadClient.pauseUpload();
151+
uploadClient.pauseUpload(contribution.getPageId());
152152
contribution.setState(Contribution.STATE_PAUSED);
153153
compositeDisposable.add(contributionDao.update(contribution)
154154
.subscribeOn(ioThreadScheduler)
@@ -312,8 +312,6 @@ private void uploadContribution(Contribution contribution) {
312312
.flatMap(uploadStash -> {
313313
notificationManager.cancel(notificationTag, NOTIFICATION_UPLOAD_IN_PROGRESS);
314314

315-
Timber.d("Stash upload response 1 is %s", uploadStash.toString());
316-
317315
if (uploadStash.getState() == StashUploadState.SUCCESS) {
318316
Timber.d("making sure of uniqueness of name: %s", filename);
319317
String uniqueFilename = findUniqueFilename(filename);
@@ -332,7 +330,6 @@ public void accept(Throwable throwable) throws Exception {
332330
}
333331
});
334332
} else if (uploadStash.getState() == StashUploadState.PAUSED) {
335-
Timber.d("Contribution upload paused");
336333
showPausedNotification(contribution);
337334
return Observable.never();
338335
} else {
@@ -359,7 +356,6 @@ private void clearChunks(Contribution contribution) {
359356

360357
private void onUpload(Contribution contribution, String notificationTag,
361358
UploadResult uploadResult) {
362-
Timber.d("Stash upload response 2 is %s", uploadResult.toString());
363359

364360
notificationManager.cancel(notificationTag, NOTIFICATION_UPLOAD_IN_PROGRESS);
365361

@@ -401,7 +397,7 @@ private void saveCompletedContribution(Contribution contribution, UploadResult u
401397

402398
@SuppressLint("StringFormatInvalid")
403399
@SuppressWarnings("deprecation")
404-
private void showFailedNotification(Contribution contribution) {
400+
private void showFailedNotification(final Contribution contribution) {
405401
final String displayTitle = contribution.getMedia().getDisplayTitle();
406402
curNotification.setTicker(getString(R.string.upload_failed_notification_title, displayTitle))
407403
.setContentTitle(getString(R.string.upload_failed_notification_title, displayTitle))
@@ -412,14 +408,15 @@ private void showFailedNotification(Contribution contribution) {
412408
curNotification.build());
413409

414410
contribution.setState(Contribution.STATE_FAILED);
411+
contribution.setChunkInfo(null);
415412

416413
compositeDisposable.add(contributionDao
417414
.update(contribution)
418415
.subscribeOn(ioThreadScheduler)
419416
.subscribe());
420417
}
421418

422-
private void showPausedNotification(Contribution contribution) {
419+
private void showPausedNotification(final Contribution contribution) {
423420
final String displayTitle = contribution.getMedia().getDisplayTitle();
424421
curNotification.setTicker(getString(R.string.upload_paused_notification_title, displayTitle))
425422
.setContentTitle(getString(R.string.upload_paused_notification_title, displayTitle))

‎build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ buildscript {
77
maven { url "https://plugins.gradle.org/m2/" }
88
}
99
dependencies {
10-
classpath 'com.android.tools.build:gradle:4.0.0'
10+
classpath 'com.android.tools.build:gradle:4.0.1'
1111
classpath "com.hiya:jacoco-android:0.2"
1212
classpath 'com.getkeepsafe.dexcount:dexcount-gradle-plugin:0.8.2'
1313
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$KOTLIN_VERSION"

0 commit comments

Comments
 (0)