Adding SHA-1 hash validation support.

extensions/data-transfer/portability-data-transfer-backblaze/src/test/java/org/datatransferproject/datatransfer/backblaze/photos/BackblazePhotosImporterTest.java

@@ -112,7 +112,7 @@ public void testImportPhoto() throws Exception {
  String albumId = "albumId";
  String response = "response";
  UUID jobId = UUID.randomUUID();
- PhotoModel photoModel = new PhotoModel(title, photoUrl, "", "", dataId, albumId, false, null);
+ PhotoModel photoModel = new PhotoModel(title, photoUrl, "", "", dataId, albumId, false);
  PhotosContainerResource data = new PhotosContainerResource(Collections.emptyList(), Collections.singletonList(photoModel));
 
  when(executor.getCachedValue(albumId)).thenReturn(albumName);

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/photos/GooglePhotosExporter.java

@@ -45,6 +45,7 @@
 import org.datatransferproject.spi.transfer.types.InvalidTokenException;
 import org.datatransferproject.spi.transfer.types.PermissionDeniedException;
 import org.datatransferproject.spi.transfer.types.TempPhotosData;
+import org.datatransferproject.spi.transfer.types.UploadErrorException;
 import org.datatransferproject.types.common.ExportInformation;
 import org.datatransferproject.types.common.PaginationData;
 import org.datatransferproject.types.common.StringPaginationToken;
@@ -98,7 +99,7 @@ public GooglePhotosExporter(
  @Override
  public ExportResult<PhotosContainerResource> export(
  UUID jobId, TokensAndUrlAuthData authData, Optional<ExportInformation> exportInformation)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  if (!exportInformation.isPresent()) {
  // Make list of photos contained in albums so they are not exported twice later on
  populateContainedPhotosList(jobId, authData);
@@ -233,7 +234,7 @@ ExportResult<PhotosContainerResource> exportPhotos(
  Optional<IdOnlyContainerResource> albumData,
  Optional<PaginationData> paginationData,
  UUID jobId)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  Optional<String> albumId = Optional.empty();
  if (albumData.isPresent()) {
  albumId = Optional.of(albumData.get().getId());
@@ -271,7 +272,7 @@ ExportResult<PhotosContainerResource> exportPhotos(
  */
  @VisibleForTesting
  void populateContainedPhotosList(UUID jobId, TokensAndUrlAuthData authData)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // This method is only called once at the beginning of the transfer, so we can start by
  // initializing a new TempPhotosData to be store in the job store.
  TempPhotosData tempPhotosData = new TempPhotosData(jobId);

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/photos/GooglePhotosImporter.java

@@ -16,6 +16,7 @@
 package org.datatransferproject.datatransfer.google.photos;
 
 import static java.lang.String.format;
+import static org.datatransferproject.datatransfer.google.photos.GooglePhotosInterface.ERROR_HASH_MISMATCH;
 
 import com.google.api.client.auth.oauth2.Credential;
 import com.google.api.client.json.JsonFactory;
@@ -57,6 +58,7 @@
 import org.datatransferproject.spi.transfer.types.DestinationMemoryFullException;
 import org.datatransferproject.spi.transfer.types.InvalidTokenException;
 import org.datatransferproject.spi.transfer.types.PermissionDeniedException;
+import org.datatransferproject.spi.transfer.types.UploadErrorException;
 import org.datatransferproject.transfer.ImageStreamProvider;
 import org.datatransferproject.types.common.ImportableItem;
 import org.datatransferproject.types.common.models.photos.PhotoAlbum;
@@ -142,7 +144,7 @@ public ImportResult importItem(
 
  @VisibleForTesting
  String importSingleAlbum(UUID jobId, TokensAndUrlAuthData authData, PhotoAlbum inputAlbum)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // Set up album
  GoogleAlbum googleAlbum = new GoogleAlbum();
  String title = Strings.nullToEmpty(inputAlbum.getName());
@@ -201,7 +203,7 @@ long importPhotos(
  return bytes;
  }
 
- long importPhotoBatch(
+ private long importPhotoBatch(
  UUID jobId,
  TokensAndUrlAuthData authData,
  List<PhotoModel> photos,
@@ -222,11 +224,20 @@ long importPhotoBatch(
  getInputStreamForUrl(jobId, photo.getFetchableUrl(), photo.isInTempStore());
 
  try (InputStream s = inputStreamBytesPair.getLeft()) {
- String uploadToken = getOrCreatePhotosInterface(jobId, authData).uploadPhotoContent(s);
+ String uploadToken = getOrCreatePhotosInterface(jobId, authData).uploadPhotoContent(s,
+ photo.getSha1());
  mediaItems.add(new NewMediaItem(cleanDescription(photo.getDescription()), uploadToken));
  uploadTokenToDataId.put(uploadToken, photo);
  size = inputStreamBytesPair.getRight();
  uploadTokenToLength.put(uploadToken, size);
+ } catch (UploadErrorException e) {
+ if (e.getMessage().contains(ERROR_HASH_MISMATCH)) {
+ monitor.severe(
+ () -> format("%s: SHA-1 (%s) mismatch during upload", jobId, photo.getSha1()));
+ }
+
+ Long finalSize = size;
+ executor.importAndSwallowIOExceptions(photo, p -> ItemImportResult.error(e, finalSize));
  }
 
  try {

extensions/data-transfer/portability-data-transfer-google/src/main/java/org/datatransferproject/datatransfer/google/photos/GooglePhotosInterface.java

@@ -36,20 +36,23 @@
 import com.google.common.base.Charsets;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.io.BaseEncoding;
 import com.google.common.io.CharStreams;
 import com.google.common.util.concurrent.RateLimiter;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
+import java.util.Base64;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 import org.datatransferproject.api.launcher.Monitor;
 import org.datatransferproject.datatransfer.google.common.GoogleCredentialFactory;
 import org.datatransferproject.datatransfer.google.mediaModels.AlbumListResponse;
@@ -60,9 +63,12 @@
 import org.datatransferproject.datatransfer.google.mediaModels.NewMediaItemUpload;
 import org.datatransferproject.spi.transfer.types.InvalidTokenException;
 import org.datatransferproject.spi.transfer.types.PermissionDeniedException;
+import org.datatransferproject.spi.transfer.types.UploadErrorException;
 
 public class GooglePhotosInterface {
 
+ public static final String ERROR_HASH_MISMATCH = "Hash mismatch";
+
  private static final String BASE_URL = "https://photoslibrary.googleapis.com/v1/";
  private static final int ALBUM_PAGE_SIZE = 20; // TODO
  private static final int MEDIA_PAGE_SIZE = 50; // TODO
@@ -122,7 +128,7 @@ GoogleMediaItem getMediaItem(String mediaId) throws IOException, InvalidTokenExc
  }
 
  MediaItemSearchResponse listMediaItems(Optional<String> albumId, Optional<String> pageToken)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  Map<String, Object> params = new LinkedHashMap<>();
  params.put(PAGE_SIZE_KEY, String.valueOf(MEDIA_PAGE_SIZE));
  if (albumId.isPresent()) {
@@ -134,21 +140,22 @@ MediaItemSearchResponse listMediaItems(Optional<String> albumId, Optional<String
  params.put(TOKEN_KEY, pageToken.get());
  }
  HttpContent content = new JsonHttpContent(this.jsonFactory, params);
- return makePostRequest(
- BASE_URL + "mediaItems:search", Optional.empty(), content, MediaItemSearchResponse.class);
+ return makePostRequest(BASE_URL + "mediaItems:search", Optional.empty(), Optional.empty(),
+ content, MediaItemSearchResponse.class);
  }
 
  GoogleAlbum createAlbum(GoogleAlbum googleAlbum)
-  throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  Map<String, Object> albumMap = createJsonMap(googleAlbum);
  Map<String, Object> contentMap = ImmutableMap.of("album", albumMap);
  HttpContent content = new JsonHttpContent(jsonFactory, contentMap);
 
- return makePostRequest(BASE_URL + "albums", Optional.empty(), content, GoogleAlbum.class);
+ return makePostRequest(BASE_URL + "albums", Optional.empty(), Optional.empty(), content,
+ GoogleAlbum.class);
  }
 
- String uploadPhotoContent(InputStream inputStream)
-  throws IOException, InvalidTokenException, PermissionDeniedException {
+ String uploadPhotoContent(InputStream inputStream, @Nullable String sha1)
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // TODO: add filename
  InputStreamContent content = new InputStreamContent(null, inputStream);
  ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@@ -160,24 +167,31 @@ String uploadPhotoContent(InputStream inputStream)
  }
  HttpContent httpContent = new ByteArrayContent(null, contentBytes);
 
- return makePostRequest(
- BASE_URL + "uploads/", Optional.of(PHOTO_UPLOAD_PARAMS), httpContent, String.class);
+ // Adding optional fields.
+ ImmutableMap.Builder<String, String> headers = ImmutableMap.builder();
+ if (sha1 != null && !sha1.isEmpty()) {
+ // Running a very naive pre-check on the string format.
+ Preconditions.checkState(sha1.length() == 40, "Invalid SHA-1 string.");
+ // Note that the base16 encoder only accepts upper cases.
+ headers.put("X-Goog-Hash", "sha1=" + Base64.getEncoder()
+ .encodeToString(BaseEncoding.base16().decode(sha1.toUpperCase())));
+ }
+
+ return makePostRequest(BASE_URL + "uploads/", Optional.of(PHOTO_UPLOAD_PARAMS),
+ Optional.of(headers.build()), httpContent, String.class);
  }
 
  BatchMediaItemResponse createPhotos(NewMediaItemUpload newMediaItemUpload)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  HashMap<String, Object> map = createJsonMap(newMediaItemUpload);
  HttpContent httpContent = new JsonHttpContent(this.jsonFactory, map);
 
- return makePostRequest(
- BASE_URL + "mediaItems:batchCreate",
- Optional.empty(),
- httpContent,
- BatchMediaItemResponse.class);
+ return makePostRequest(BASE_URL + "mediaItems:batchCreate", Optional.empty(), Optional.empty(),
+ httpContent, BatchMediaItemResponse.class);
  }
 
  private <T> T makeGetRequest(String url, Optional<Map<String, String>> parameters, Class<T> clazz)
-  throws IOException, InvalidTokenException, PermissionDeniedException {
+ throws IOException, InvalidTokenException, PermissionDeniedException {
  HttpRequestFactory requestFactory = httpTransport.createRequestFactory();
  HttpRequest getRequest =
  requestFactory.buildGetRequest(
@@ -201,22 +215,26 @@ private <T> T makeGetRequest(String url, Optional<Map<String, String>> parameter
  return objectMapper.readValue(result, clazz);
  }
 
- <T> T makePostRequest(
- String url, Optional<Map<String, String>> parameters, HttpContent httpContent, Class<T> clazz)
- throws IOException, InvalidTokenException, PermissionDeniedException {
+ <T> T makePostRequest(String url, Optional<Map<String, String>> parameters,
+ Optional<Map<String, String>> extraHeaders, HttpContent httpContent, Class<T> clazz)
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // Wait for write permit before making request
  writeRateLimiter.acquire();
 
  HttpRequestFactory requestFactory = httpTransport.createRequestFactory();
  HttpRequest postRequest =
  requestFactory.buildPostRequest(
  new GenericUrl(url + "?" + generateParamsString(parameters)), httpContent);
+ extraHeaders.ifPresent(stringStringMap -> stringStringMap.forEach(
+ (key, value) -> postRequest.getHeaders().set(key, value)));
  postRequest.setReadTimeout(2 * 60000); // 2 minutes read timeout
  HttpResponse response;
 
  try {
  response = postRequest.execute();
  } catch (HttpResponseException e) {
+ maybeRethrowAsUploadError(e);
+
  response =
  handleHttpResponseException(
  () ->
@@ -235,6 +253,19 @@ <T> T makePostRequest(
  }
  }
 
+ /**
+ * Converting {@link HttpResponseException} to upload-related exceptions. Current this is only
+ * used for payload hash verifications.
+ *
+ * Note that making this a separate method to avoid polluting throw lists.
+ */
+ private void maybeRethrowAsUploadError(HttpResponseException e) throws UploadErrorException {
+ if (e.getStatusCode() == 400 && e.getContent()
+ .contains("Checksum from header does not match received payload content.")) {
+ throw new UploadErrorException(ERROR_HASH_MISMATCH, e);
+ }
+ }
+
  private HttpResponse handleHttpResponseException(
  SupplierWithIO<HttpRequest> httpRequest, HttpResponseException e)
  throws IOException, InvalidTokenException, PermissionDeniedException {
@@ -246,7 +277,7 @@ private HttpResponse handleHttpResponseException(
  // if the credential refresh failed, let the error bubble up via the IOException that gets
  // thrown
  credential = credentialFactory.refreshCredential(credential);
- monitor.info(() -> "Refreshed authorization token successfuly");
+ monitor.info(() -> "Refreshed authorization token successfully");
 
  // if the second attempt throws an error, then something else is wrong, and we bubble up the
  // response errors

extensions/data-transfer/portability-data-transfer-google/src/test/java/org/datatransferproject/datatransfer/google/photos/GooglePhotosExporterTest.java

@@ -51,6 +51,7 @@
 import org.datatransferproject.spi.transfer.types.InvalidTokenException;
 import org.datatransferproject.spi.transfer.types.PermissionDeniedException;
 import org.datatransferproject.spi.transfer.types.TempPhotosData;
+import org.datatransferproject.spi.transfer.types.UploadErrorException;
 import org.datatransferproject.types.common.PaginationData;
 import org.datatransferproject.types.common.StringPaginationToken;
 import org.datatransferproject.types.common.models.ContainerResource;
@@ -81,7 +82,8 @@ public class GooglePhotosExporterTest {
  private AlbumListResponse albumListResponse;
 
  @BeforeEach
- public void setup() throws IOException, InvalidTokenException, PermissionDeniedException {
+ public void setup()
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  GoogleCredentialFactory credentialFactory = mock(GoogleCredentialFactory.class);
  jobStore = mock(TemporaryPerJobDataStore.class);
  when(jobStore.getStream(any(), anyString())).thenReturn(mock(InputStreamWrapper.class));
@@ -166,7 +168,8 @@ public void exportAlbumSubsequentSet() throws IOException, InvalidTokenException
  }
 
  @Test
- public void exportPhotoFirstSet() throws IOException, InvalidTokenException, PermissionDeniedException {
+ public void exportPhotoFirstSet()
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  setUpSingleAlbum();
  when(albumListResponse.getNextPageToken()).thenReturn(null);
  GoogleMediaItem mediaItem = setUpSinglePhoto(IMG_URI, PHOTO_ID);
@@ -205,7 +208,8 @@ public void exportPhotoFirstSet() throws IOException, InvalidTokenException, Per
  }
 
  @Test
- public void exportPhotoSubsequentSet() throws IOException, InvalidTokenException, PermissionDeniedException {
+ public void exportPhotoSubsequentSet()
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  setUpSingleAlbum();
  when(albumListResponse.getNextPageToken()).thenReturn(null);
  GoogleMediaItem mediaItem = setUpSinglePhoto(IMG_URI, PHOTO_ID);
@@ -233,7 +237,8 @@ public void exportPhotoSubsequentSet() throws IOException, InvalidTokenException
  }
 
  @Test
- public void populateContainedPhotosList() throws IOException, InvalidTokenException, PermissionDeniedException {
+ public void populateContainedPhotosList()
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // Set up an album with two photos
  setUpSingleAlbum();
  when(albumListResponse.getNextPageToken()).thenReturn(null);
@@ -266,7 +271,8 @@ public void populateContainedPhotosList() throws IOException, InvalidTokenExcept
  /* Tests that when there is no album information passed along to exportPhotos, only albumless
  photos are exported.
  */
- public void onlyExportAlbumlessPhoto() throws IOException, InvalidTokenException, PermissionDeniedException {
+ public void onlyExportAlbumlessPhoto()
+ throws IOException, InvalidTokenException, PermissionDeniedException, UploadErrorException {
  // Set up - two photos will be returned by a media item search without an album id, but one of
  // them will have already been put into the list of contained photos
  String containedPhotoUri = "contained photo uri";