Skip to content

Commit f0a1d03

Browse files
pshpsh
authored
Convert upload client to kotlin (commons-app#5568)
* Write unit tests for the UploadClient (with gentle refactoring to make it testable) * Convert Upload client to kotlin
1 parent 728712c commit f0a1d03

File tree

10 files changed

+552
-247
lines changed

10 files changed

+552
-247
lines changed

‎app/src/main/java/fr/free/nrw/commons/contributions/Contribution.kt

+18-1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.os.Parcelable
55
import androidx.room.Embedded
66
import androidx.room.Entity
77
import androidx.room.PrimaryKey
8+
import fr.free.nrw.commons.CommonsApplication
89
import fr.free.nrw.commons.Media
910
import fr.free.nrw.commons.auth.SessionManager
1011
import fr.free.nrw.commons.upload.UploadItem
@@ -13,7 +14,8 @@ import fr.free.nrw.commons.upload.WikidataPlace
1314
import fr.free.nrw.commons.upload.WikidataPlace.Companion.from
1415
import fr.free.nrw.commons.upload.structure.depictions.DepictedItem
1516
import kotlinx.parcelize.Parcelize
16-
import java.util.*
17+
import java.io.File
18+
import java.util.Date
1719

1820
@Entity(tableName = "contribution")
1921
@Parcelize
@@ -117,4 +119,19 @@ data class Contribution constructor(
117119
descriptions.filter { it.descriptionText.isNotEmpty() }
118120
.joinToString(separator = "") { "{{${it.languageCode}|1=${it.descriptionText}}}" }
119121
}
122+
123+
val fileKey : String? get() = chunkInfo?.uploadResult?.filekey
124+
val localUriPath: File? get() = localUri?.path?.let { File(it) }
125+
126+
fun isCompleted(): Boolean {
127+
return chunkInfo != null && chunkInfo!!.totalChunks == chunkInfo!!.indexOfNextChunkToUpload
128+
}
129+
130+
fun isPaused(): Boolean {
131+
return CommonsApplication.pauseUploads[pageId] ?: false
132+
}
133+
134+
fun unpause() {
135+
CommonsApplication.pauseUploads[pageId] = false
136+
}
120137
}

‎app/src/main/java/fr/free/nrw/commons/upload/CountingRequestBody.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ class CountingRequestBody(
5252
}
5353
}
5454

55-
interface Listener {
55+
fun interface Listener {
5656
/**
5757
* Will be triggered when write progresses
5858
* @param bytesWritten

‎app/src/main/java/fr/free/nrw/commons/upload/FileUtilsWrapper.java

+15-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package fr.free.nrw.commons.upload;
22

33
import android.content.Context;
4+
import android.net.Uri;
45
import fr.free.nrw.commons.location.LatLng;
56
import io.reactivex.Observable;
67
import java.io.BufferedInputStream;
@@ -21,9 +22,11 @@
2122
@Singleton
2223
public class FileUtilsWrapper {
2324

24-
@Inject
25-
public FileUtilsWrapper() {
25+
private final Context context;
2626

27+
@Inject
28+
public FileUtilsWrapper(final Context context) {
29+
this.context = context;
2730
}
2831

2932
public String getFileExt(String fileName) {
@@ -42,11 +45,18 @@ public String getGeolocationOfFile(String filePath, LatLng inAppPictureLocation)
4245
return FileUtils.getGeolocationOfFile(filePath, inAppPictureLocation);
4346
}
4447

48+
public String getMimeType(File file) {
49+
return getMimeType(Uri.parse(file.getPath()));
50+
}
51+
52+
public String getMimeType(Uri uri) {
53+
return FileUtils.getMimeType(context, uri);
54+
}
4555

4656
/**
4757
* Takes a file as input and returns an Observable of files with the specified chunk size
4858
*/
49-
public List<File> getFileChunks(Context context, File file, final int chunkSize)
59+
public List<File> getFileChunks(File file, final int chunkSize)
5060
throws IOException {
5161
final byte[] buffer = new byte[chunkSize];
5262

@@ -56,7 +66,7 @@ public List<File> getFileChunks(Context context, File file, final int chunkSize)
5666
final List<File> buffers = new ArrayList<>();
5767
int size;
5868
while ((size = bis.read(buffer)) > 0) {
59-
buffers.add(writeToFile(context, Arrays.copyOf(buffer, size), file.getName(),
69+
buffers.add(writeToFile(Arrays.copyOf(buffer, size), file.getName(),
6070
getFileExt(file.getName())));
6171
}
6272
return buffers;
@@ -66,7 +76,7 @@ public List<File> getFileChunks(Context context, File file, final int chunkSize)
6676
/**
6777
* Create a temp file containing the passed byte data.
6878
*/
69-
private File writeToFile(Context context, final byte[] data, final String fileName,
79+
private File writeToFile(final byte[] data, final String fileName,
7080
String fileExtension)
7181
throws IOException {
7282
final File file = File.createTempFile(fileName, fileExtension, context.getCacheDir());

‎app/src/main/java/fr/free/nrw/commons/upload/PageContentsCreator.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
import org.apache.commons.lang3.StringUtils;
1818
import timber.log.Timber;
1919

20-
class PageContentsCreator {
20+
public class PageContentsCreator {
2121

2222
//{{According to Exif data|2009-01-09}}
2323
private static final String TEMPLATE_DATE_ACC_TO_EXIF = "{{According to Exif data|%s}}";

‎app/src/main/java/fr/free/nrw/commons/upload/UploadClient.java

-236
Original file line numberDiff line numberDiff line change
@@ -1,236 +0,0 @@
1-
package fr.free.nrw.commons.upload;
2-
3-
import static fr.free.nrw.commons.di.NetworkingModule.NAMED_COMMONS_CSRF;
4-
5-
import android.content.Context;
6-
import android.net.Uri;
7-
import androidx.annotation.Nullable;
8-
import com.google.gson.Gson;
9-
import fr.free.nrw.commons.CommonsApplication;
10-
import fr.free.nrw.commons.contributions.ChunkInfo;
11-
import fr.free.nrw.commons.contributions.Contribution;
12-
import fr.free.nrw.commons.upload.worker.UploadWorker.NotificationUpdateProgressListener;
13-
import io.reactivex.Observable;
14-
import io.reactivex.disposables.CompositeDisposable;
15-
import java.io.File;
16-
import java.io.IOException;
17-
import java.net.URLEncoder;
18-
import java.util.Date;
19-
import java.util.List;
20-
import java.util.concurrent.atomic.AtomicBoolean;
21-
import java.util.concurrent.atomic.AtomicInteger;
22-
import java.util.concurrent.atomic.AtomicReference;
23-
import javax.inject.Inject;
24-
import javax.inject.Named;
25-
import javax.inject.Singleton;
26-
import okhttp3.MediaType;
27-
import okhttp3.MultipartBody;
28-
import okhttp3.RequestBody;
29-
import fr.free.nrw.commons.auth.csrf.CsrfTokenClient;
30-
import fr.free.nrw.commons.wikidata.mwapi.MwException;
31-
import timber.log.Timber;
32-
33-
@Singleton
34-
public class UploadClient {
35-
36-
private final int CHUNK_SIZE = 512 * 1024; // 512 KB
37-
38-
//This is maximum duration for which a stash is persisted on MediaWiki
39-
// https://www.mediawiki.org/wiki/Manual:$wgUploadStashMaxAge
40-
private final int MAX_CHUNK_AGE = 6 * 3600 * 1000; // 6 hours
41-
42-
private final UploadInterface uploadInterface;
43-
private final CsrfTokenClient csrfTokenClient;
44-
private final PageContentsCreator pageContentsCreator;
45-
private final FileUtilsWrapper fileUtilsWrapper;
46-
private final Gson gson;
47-
48-
private final CompositeDisposable compositeDisposable = new CompositeDisposable();
49-
50-
@Inject
51-
public UploadClient(final UploadInterface uploadInterface,
52-
@Named(NAMED_COMMONS_CSRF) final CsrfTokenClient csrfTokenClient,
53-
final PageContentsCreator pageContentsCreator,
54-
final FileUtilsWrapper fileUtilsWrapper, final Gson gson) {
55-
this.uploadInterface = uploadInterface;
56-
this.csrfTokenClient = csrfTokenClient;
57-
this.pageContentsCreator = pageContentsCreator;
58-
this.fileUtilsWrapper = fileUtilsWrapper;
59-
this.gson = gson;
60-
}
61-
62-
/**
63-
* Upload file to stash in chunks of specified size. Uploading files in chunks will make
64-
* handling of large files easier. Also, it will be useful in supporting pause/resume of
65-
* uploads
66-
*/
67-
public Observable<StashUploadResult> uploadFileToStash(
68-
final Context context, final String filename, final Contribution contribution,
69-
final NotificationUpdateProgressListener notificationUpdater) throws IOException {
70-
if (contribution.getChunkInfo() != null
71-
&& contribution.getChunkInfo().getTotalChunks() == contribution.getChunkInfo()
72-
.getIndexOfNextChunkToUpload()) {
73-
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
74-
contribution.getChunkInfo().getUploadResult().getFilekey()));
75-
}
76-
77-
CommonsApplication.pauseUploads.put(contribution.getPageId(), false);
78-
79-
final File file = new File(contribution.getLocalUri().getPath());
80-
final List<File> fileChunks = fileUtilsWrapper.getFileChunks(context, file, CHUNK_SIZE);
81-
82-
final int totalChunks = fileChunks.size();
83-
84-
final MediaType mediaType = MediaType
85-
.parse(FileUtils.getMimeType(context, Uri.parse(file.getPath())));
86-
87-
final AtomicReference<ChunkInfo> chunkInfo = new AtomicReference<>();
88-
if (isStashValid(contribution)) {
89-
chunkInfo.set(contribution.getChunkInfo());
90-
91-
Timber.d("Chunk: Next Chunk: %s, Total Chunks: %s",
92-
contribution.getChunkInfo().getIndexOfNextChunkToUpload(),
93-
contribution.getChunkInfo().getTotalChunks());
94-
}
95-
96-
final AtomicInteger index = new AtomicInteger();
97-
final AtomicBoolean failures = new AtomicBoolean();
98-
99-
compositeDisposable.add(Observable.fromIterable(fileChunks).forEach(chunkFile -> {
100-
if (CommonsApplication.pauseUploads.get(contribution.getPageId()) || failures.get()) {
101-
return;
102-
}
103-
104-
if (chunkInfo.get() != null && index.get() < chunkInfo.get()
105-
.getIndexOfNextChunkToUpload()) {
106-
index.incrementAndGet();
107-
Timber.d("Chunk: Increment and return: %s", index.get());
108-
return;
109-
}
110-
index.getAndIncrement();
111-
final int offset =
112-
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getOffset() : 0;
113-
114-
Timber.d("Chunk: Sending Chunk number: %s, offset: %s", index.get(), offset);
115-
final String filekey =
116-
chunkInfo.get() != null ? chunkInfo.get().getUploadResult().getFilekey() : null;
117-
118-
final RequestBody requestBody = RequestBody
119-
.create(mediaType, chunkFile);
120-
final CountingRequestBody countingRequestBody = new CountingRequestBody(requestBody,
121-
notificationUpdater::onProgress, offset,
122-
file.length());
123-
124-
compositeDisposable.add(uploadChunkToStash(filename,
125-
file.length(),
126-
offset,
127-
filekey,
128-
countingRequestBody).subscribe(uploadResult -> {
129-
Timber.d("Chunk: Received Chunk number: %s, offset: %s", index.get(),
130-
uploadResult.getOffset());
131-
chunkInfo.set(
132-
new ChunkInfo(uploadResult, index.get(), totalChunks));
133-
notificationUpdater.onChunkUploaded(contribution, chunkInfo.get());
134-
}, throwable -> {
135-
Timber.e(throwable, "Received error in chunk upload");
136-
failures.set(true);
137-
}));
138-
}));
139-
140-
if (CommonsApplication.pauseUploads.get(contribution.getPageId())) {
141-
Timber.d("Upload stash paused %s", contribution.getPageId());
142-
return Observable.just(new StashUploadResult(StashUploadState.PAUSED, null));
143-
} else if (failures.get()) {
144-
Timber.d("Upload stash contains failures %s", contribution.getPageId());
145-
return Observable.just(new StashUploadResult(StashUploadState.FAILED, null));
146-
} else if (chunkInfo.get() != null) {
147-
Timber.d("Upload stash success %s", contribution.getPageId());
148-
return Observable.just(new StashUploadResult(StashUploadState.SUCCESS,
149-
chunkInfo.get().getUploadResult().getFilekey()));
150-
} else {
151-
Timber.d("Upload stash failed %s", contribution.getPageId());
152-
return Observable.just(new StashUploadResult(StashUploadState.FAILED, null));
153-
}
154-
}
155-
156-
/**
157-
* Stash is valid for 6 hours. This function checks the validity of stash
158-
*
159-
* @param contribution
160-
* @return
161-
*/
162-
private boolean isStashValid(Contribution contribution) {
163-
return contribution.getChunkInfo() != null &&
164-
contribution.getDateModified()
165-
.after(new Date(System.currentTimeMillis() - MAX_CHUNK_AGE));
166-
}
167-
168-
/**
169-
* Uploads a file chunk to stash
170-
*
171-
* @param filename The name of the file being uploaded
172-
* @param fileSize The total size of the file
173-
* @param offset The offset returned by the previous chunk upload
174-
* @param fileKey The filekey returned by the previous chunk upload
175-
* @param countingRequestBody Request body with chunk file
176-
* @return
177-
*/
178-
Observable<UploadResult> uploadChunkToStash(final String filename,
179-
final long fileSize,
180-
final long offset,
181-
final String fileKey,
182-
final CountingRequestBody countingRequestBody) {
183-
final MultipartBody.Part filePart;
184-
try {
185-
filePart = MultipartBody.Part
186-
.createFormData("chunk", URLEncoder.encode(filename, "utf-8"), countingRequestBody);
187-
188-
return uploadInterface.uploadFileToStash(toRequestBody(filename),
189-
toRequestBody(String.valueOf(fileSize)),
190-
toRequestBody(String.valueOf(offset)),
191-
toRequestBody(fileKey),
192-
toRequestBody(csrfTokenClient.getTokenBlocking()),
193-
filePart)
194-
.map(UploadResponse::getUpload);
195-
} catch (final Throwable throwable) {
196-
Timber.e(throwable, "Failed to upload chunk to stash");
197-
return Observable.error(throwable);
198-
}
199-
}
200-
201-
/**
202-
* Converts string value to request body
203-
*/
204-
@Nullable
205-
private RequestBody toRequestBody(@Nullable final String value) {
206-
return value == null ? null : RequestBody.create(okhttp3.MultipartBody.FORM, value);
207-
}
208-
209-
210-
public Observable<UploadResult> uploadFileFromStash(
211-
final Contribution contribution,
212-
final String uniqueFileName,
213-
final String fileKey) {
214-
try {
215-
return uploadInterface
216-
.uploadFileFromStash(csrfTokenClient.getTokenBlocking(),
217-
pageContentsCreator.createFrom(contribution),
218-
CommonsApplication.DEFAULT_EDIT_SUMMARY,
219-
uniqueFileName,
220-
fileKey).map(uploadResponse -> {
221-
final UploadResponse uploadResult = gson
222-
.fromJson(uploadResponse, UploadResponse.class);
223-
if (uploadResult.getUpload() == null) {
224-
final MwException exception = gson
225-
.fromJson(uploadResponse, MwException.class);
226-
Timber.e(exception, "Error in uploading file from stash");
227-
throw new Exception(exception.getErrorCode());
228-
}
229-
return uploadResult.getUpload();
230-
});
231-
} catch (final Throwable throwable) {
232-
Timber.e(throwable, "Exception occurred in uploading file from stash");
233-
return Observable.error(throwable);
234-
}
235-
}
236-
}

0 commit comments

Comments
 (0)