diff --git a/.codegen.json b/.codegen.json index 247315aff..16a99a898 100644 --- a/.codegen.json +++ b/.codegen.json @@ -1 +1 @@ -{ "engineHash": "3d7ee46", "specHash": "88cd5aa", "version": "5.16.0" } +{ "engineHash": "0072e8b", "specHash": "88cd5aa", "version": "5.16.0" } diff --git a/docs/sdkgen/chunkeduploads.md b/docs/sdkgen/chunkeduploads.md index fc4d1a6ca..77bffd1eb 100644 --- a/docs/sdkgen/chunkeduploads.md +++ b/docs/sdkgen/chunkeduploads.md @@ -17,6 +17,7 @@ This is a manager for chunked uploads (allowed for files at least 20MB). - [Commit upload session by URL](#commit-upload-session-by-url) - [Commit upload session](#commit-upload-session) - [Upload big file](#upload-big-file) +- [Upload big file version](#upload-big-file-version) ## Create upload session @@ -524,3 +525,34 @@ This function returns a value of type `FileFull`. +## Upload big file version + +Starts the process of chunk uploading a new version of a big file. Should return a File object representing the uploaded file version. Returns nothing when commit responds with 202 because the file did not change. + +This operation is performed by calling function `uploadBigFileVersion`. + + + +``` +client.getChunkedUploads().uploadBigFileVersion(uploadedFile.getId(), generateByteStream(versionFileSize), versionFileSize, versionName) +``` + +### Arguments + +- fileId `String` + - The ID of the file to upload a new version of. +- file `InputStream` + - The stream of the file to upload. +- fileSize `long` + - The total size of the file for the chunked upload in bytes. +- fileName `String` + - The optional new name of the file. + + +### Returns + +This function returns a value of type `FileFull`. + + + + diff --git a/src/intTest/java/com/box/sdkgen/chunkeduploads/ChunkedUploadsITest.java b/src/intTest/java/com/box/sdkgen/chunkeduploads/ChunkedUploadsITest.java index c6c16080f..e2a6919db 100644 --- a/src/intTest/java/com/box/sdkgen/chunkeduploads/ChunkedUploadsITest.java +++ b/src/intTest/java/com/box/sdkgen/chunkeduploads/ChunkedUploadsITest.java @@ -24,6 +24,7 @@ import com.box.sdkgen.managers.chunkeduploads.UploadFilePartByUrlHeaders; import com.box.sdkgen.managers.chunkeduploads.UploadFilePartHeaders; import com.box.sdkgen.schemas.file.File; +import com.box.sdkgen.schemas.filefull.FileFull; import com.box.sdkgen.schemas.files.Files; import com.box.sdkgen.schemas.uploadedpart.UploadedPart; import com.box.sdkgen.schemas.uploadpart.UploadPart; @@ -309,4 +310,34 @@ public void testChunkedUploadConvenienceMethod() { assert uploadedFile.getParent().getId().equals(parentFolderId); client.getFiles().deleteFileById(uploadedFile.getId()); } + + @Test + public void testChunkedUploadFileVersionConvenienceMethod() { + String fileName = getUuid(); + int fileSize = 20 * 1024 * 1024; + String parentFolderId = "0"; + File uploadedFile = + client + .getChunkedUploads() + .uploadBigFile(generateByteStream(fileSize), fileName, fileSize, parentFolderId); + assert uploadedFile.getName().equals(fileName); + assert uploadedFile.getSize() == fileSize; + int versionFileSize = 21 * 1024 * 1024; + String versionName = getUuid(); + FileFull uploadedFileVersion = + client + .getChunkedUploads() + .uploadBigFileVersion( + uploadedFile.getId(), + generateByteStream(versionFileSize), + versionFileSize, + versionName); + assert !(uploadedFileVersion == null); + assert uploadedFileVersion.getId().equals(uploadedFile.getId()); + assert uploadedFileVersion.getName().equals(versionName); + assert uploadedFileVersion.getSize() == versionFileSize; + assert !(uploadedFileVersion.getName().equals(fileName)); + assert !(uploadedFileVersion.getSize() == uploadedFile.getSize()); + client.getFiles().deleteFileById(uploadedFile.getId()); + } } diff --git a/src/main/java/com/box/sdkgen/managers/chunkeduploads/ChunkedUploadsManager.java b/src/main/java/com/box/sdkgen/managers/chunkeduploads/ChunkedUploadsManager.java index efcc13a92..c368a70bb 100644 --- a/src/main/java/com/box/sdkgen/managers/chunkeduploads/ChunkedUploadsManager.java +++ b/src/main/java/com/box/sdkgen/managers/chunkeduploads/ChunkedUploadsManager.java @@ -23,6 +23,8 @@ import com.box.sdkgen.schemas.files.Files; import com.box.sdkgen.schemas.uploadedpart.UploadedPart; import com.box.sdkgen.schemas.uploadpart.UploadPart; +import com.box.sdkgen.schemas.uploadpartplan.UploadPartPlan; +import com.box.sdkgen.schemas.uploadpartplanhit.UploadPartPlanHit; import com.box.sdkgen.schemas.uploadparts.UploadParts; import com.box.sdkgen.schemas.uploadsession.UploadSession; import com.box.sdkgen.schemas.uploadsessionplanrequest.UploadSessionPlanRequest; @@ -826,12 +828,15 @@ public PartAccumulator reducer(PartAccumulator acc, InputStream chunk) { assert part.getSize() == chunkSize; assert part.getOffset() == bytesStart; acc.getFileHash().updateHash(chunkBuffer); - return new PartAccumulator( - bytesEnd, - Stream.concat(parts.stream(), Arrays.asList(part).stream()).collect(Collectors.toList()), - acc.getFileSize(), - acc.getUploadPartUrl(), - acc.getFileHash()); + return new PartAccumulator.Builder( + bytesEnd, + Stream.concat(parts.stream(), Arrays.asList(part).stream()) + .collect(Collectors.toList()), + acc.getFileSize(), + acc.getUploadPartUrl(), + acc.getFileHash()) + .planUrl(acc.getPlanUrl()) + .build(); } /** @@ -875,6 +880,143 @@ public FileFull uploadBigFile( return committedSession.getEntries().get(0); } + public UploadPart getCachedUploadPart(String planUrl, long offset, long size, String sha512) { + UploadSessionPlanResponse plan = + this.createFileUploadSessionPlanByUrl( + planUrl, + new UploadSessionPlanRequest(Arrays.asList(new UploadPartPlan(offset, size, sha512)))); + if (plan.getHits().size() > 0) { + UploadPartPlanHit hit = plan.getHits().get(0); + return new UploadPart.Builder() + .partId(hit.getPartId()) + .offset(hit.getOffset()) + .size(hit.getSize()) + .build(); + } + return null; + } + + public PartAccumulator reducerForFileVersion(PartAccumulator acc, InputStream chunk) { + long lastIndex = acc.getLastIndex(); + List parts = acc.getParts(); + byte[] chunkBuffer = readByteStream(chunk); + Hash hash = new Hash(HashName.SHA1); + hash.updateHash(chunkBuffer); + String sha1 = hash.digestHash("base64"); + String digest = String.join("", "sha=", sha1); + int chunkSize = bufferLength(chunkBuffer); + long bytesStart = lastIndex + 1; + long bytesEnd = lastIndex + chunkSize; + String contentRange = + String.join( + "", + "bytes ", + convertToString(bytesStart), + "-", + convertToString(bytesEnd), + "/", + convertToString(acc.getFileSize())); + Hash sha512Hash = new Hash(HashName.SHA512); + sha512Hash.updateHash(chunkBuffer); + String sha512 = sha512Hash.digestHash("hex"); + UploadPart cachedPart = + this.getCachedUploadPart(acc.getPlanUrl(), bytesStart, chunkSize, sha512); + if (!(cachedPart == null)) { + acc.getFileHash().updateHash(chunkBuffer); + return new PartAccumulator.Builder( + bytesEnd, + Stream.concat(parts.stream(), Arrays.asList(cachedPart).stream()) + .collect(Collectors.toList()), + acc.getFileSize(), + acc.getUploadPartUrl(), + acc.getFileHash()) + .planUrl(acc.getPlanUrl()) + .build(); + } + UploadedPart uploadedPart = + this.uploadFilePartByUrl( + acc.getUploadPartUrl(), + generateByteStreamFromBuffer(chunkBuffer), + new UploadFilePartByUrlHeaders(digest, contentRange)); + UploadPart part = uploadedPart.getPart(); + String partSha1 = hexToBase64(part.getSha1()); + assert partSha1.equals(sha1); + assert part.getSize() == chunkSize; + assert part.getOffset() == bytesStart; + acc.getFileHash().updateHash(chunkBuffer); + return new PartAccumulator.Builder( + bytesEnd, + Stream.concat(parts.stream(), Arrays.asList(part).stream()) + .collect(Collectors.toList()), + acc.getFileSize(), + acc.getUploadPartUrl(), + acc.getFileHash()) + .planUrl(acc.getPlanUrl()) + .build(); + } + + /** + * Starts the process of chunk uploading a new version of a big file. Should return a File object + * representing the uploaded file version. Returns nothing when commit responds with 202 because + * the file did not change. + * + * @param fileId The ID of the file to upload a new version of. + * @param file The stream of the file to upload. + * @param fileSize The total size of the file for the chunked upload in bytes. + */ + public FileFull uploadBigFileVersion(String fileId, InputStream file, long fileSize) { + return uploadBigFileVersion(fileId, file, fileSize, null); + } + + /** + * Starts the process of chunk uploading a new version of a big file. Should return a File object + * representing the uploaded file version. Returns nothing when commit responds with 202 because + * the file did not change. + * + * @param fileId The ID of the file to upload a new version of. + * @param file The stream of the file to upload. + * @param fileSize The total size of the file for the chunked upload in bytes. + * @param fileName The optional new name of the file. + */ + public FileFull uploadBigFileVersion( + String fileId, InputStream file, long fileSize, String fileName) { + UploadSession uploadSession = + this.createFileUploadSessionForExistingFile( + fileId, + new CreateFileUploadSessionForExistingFileRequestBody.Builder(fileSize) + .fileName(fileName) + .build()); + String uploadPartUrl = uploadSession.getSessionEndpoints().getUploadPart(); + String commitUrl = uploadSession.getSessionEndpoints().getCommit(); + String planUrl = uploadSession.getSessionEndpoints().getPlan(); + long partSize = uploadSession.getPartSize(); + int totalParts = uploadSession.getTotalParts(); + assert partSize * totalParts >= fileSize; + assert uploadSession.getNumPartsProcessed() == 0; + Hash fileHash = new Hash(HashName.SHA1); + Iterator chunksIterator = iterateChunks(file, partSize, fileSize); + PartAccumulator results = + reduceIterator( + chunksIterator, + this::reducerForFileVersion, + new PartAccumulator.Builder( + -1, Collections.emptyList(), fileSize, uploadPartUrl, fileHash) + .planUrl(planUrl) + .build()); + List parts = results.getParts(); + String sha1 = fileHash.digestHash("base64"); + String digest = String.join("", "sha=", sha1); + Files committedSession = + this.createFileUploadSessionCommitByUrl( + commitUrl, + new CreateFileUploadSessionCommitByUrlRequestBody(parts), + new CreateFileUploadSessionCommitByUrlHeaders(digest)); + if (committedSession == null) { + return null; + } + return committedSession.getEntries().get(0); + } + public Authentication getAuth() { return auth; } diff --git a/src/main/java/com/box/sdkgen/managers/chunkeduploads/PartAccumulator.java b/src/main/java/com/box/sdkgen/managers/chunkeduploads/PartAccumulator.java index b6c59e16a..645b634b3 100644 --- a/src/main/java/com/box/sdkgen/managers/chunkeduploads/PartAccumulator.java +++ b/src/main/java/com/box/sdkgen/managers/chunkeduploads/PartAccumulator.java @@ -16,6 +16,8 @@ public class PartAccumulator { public final Hash fileHash; + public String planUrl; + public PartAccumulator( long lastIndex, List parts, long fileSize, String uploadPartUrl, Hash fileHash) { this.lastIndex = lastIndex; @@ -23,6 +25,16 @@ public PartAccumulator( this.fileSize = fileSize; this.uploadPartUrl = uploadPartUrl; this.fileHash = fileHash; + this.planUrl = ""; + } + + protected PartAccumulator(Builder builder) { + this.lastIndex = builder.lastIndex; + this.parts = builder.parts; + this.fileSize = builder.fileSize; + this.uploadPartUrl = builder.uploadPartUrl; + this.fileHash = builder.fileHash; + this.planUrl = builder.planUrl; } public long getLastIndex() { @@ -44,4 +56,48 @@ public String getUploadPartUrl() { public Hash getFileHash() { return fileHash; } + + public String getPlanUrl() { + return planUrl; + } + + public static class Builder { + + protected final long lastIndex; + + protected final List parts; + + protected final long fileSize; + + protected final String uploadPartUrl; + + protected final Hash fileHash; + + protected String planUrl; + + public Builder( + long lastIndex, + List parts, + long fileSize, + String uploadPartUrl, + Hash fileHash) { + this.lastIndex = lastIndex; + this.parts = parts; + this.fileSize = fileSize; + this.uploadPartUrl = uploadPartUrl; + this.fileHash = fileHash; + } + + public Builder planUrl(String planUrl) { + this.planUrl = planUrl; + return this; + } + + public PartAccumulator build() { + if (this.planUrl == null) { + this.planUrl = ""; + } + return new PartAccumulator(this); + } + } }