diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java index d52a3f1dd58f..27dc7bbee035 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/context/FhirContext.java @@ -1141,12 +1141,23 @@ public String toString() { } // TODO KHS add the other primitive types + @Deprecated(since = "6.6.0", forRemoval = true) public IPrimitiveType getPrimitiveBoolean(Boolean theValue) { + return newPrimitiveBoolean(theValue); + } + + public IPrimitiveType newPrimitiveBoolean(Boolean theValue) { IPrimitiveType retval = (IPrimitiveType) getElementDefinition("boolean").newInstance(); retval.setValue(theValue); return retval; } + public IPrimitiveType newPrimitiveString(String theValue) { + IPrimitiveType retval = (IPrimitiveType) getElementDefinition("string").newInstance(); + retval.setValue(theValue); + return retval; + } + private static boolean tryToInitParser(Runnable run) { boolean retVal; try { diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java index 70a6c80a4f31..28dcec8ca965 100644 --- a/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/interceptor/api/Pointcut.java @@ -2618,6 +2618,33 @@ public enum Pointcut implements IPointcut { "ca.uhn.fhir.jpa.util.SqlQueryList" ), + /** + * Binary Blob Prefix Assigning Hook: + *

+ * Immediately before a binary blob is stored to its eventual data sink, this hook is called. + * This hook allows implementers to provide a prefix to the binary blob's ID. + * This is helpful in cases where you want to identify this blob for later retrieval outside of HAPI-FHIR. Note that allowable characters will depend on the specific storage sink being used. + *

    + *
  • + * ca.uhn.fhir.rest.api.server.RequestDetails - A bean containing details about the request that is about to be processed, including details such as the + * resource type and logical ID (if any) and other FHIR-specific aspects of the request which have been + * pulled out of the servlet request. Note that the bean + * properties are not all guaranteed to be populated. + *
    • + *
  • + * org.hl7.fhir.instance.model.api.IBaseBinary - The binary resource that is about to be stored. + *
    • + *
+ *

+ * Hooks should return String, which represents the full prefix to be applied to the blob. + *

+ */ + STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX(String.class, + "ca.uhn.fhir.rest.api.server.RequestDetails", + "org.hl7.fhir.instance.model.api.IBaseResource" + ), + + /** * This pointcut is used only for unit tests. Do not use in production code as it may be changed or * removed at any time. diff --git a/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ProxyUtil.java b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ProxyUtil.java new file mode 100644 index 000000000000..b0dd7a6e0103 --- /dev/null +++ b/hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ProxyUtil.java @@ -0,0 +1,67 @@ +package ca.uhn.fhir.util; + +/*- + * #%L + * HAPI FHIR - Core Library + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.commons.lang3.Validate; + +import java.lang.reflect.InvocationHandler; +import java.lang.reflect.Method; +import java.lang.reflect.Proxy; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkNotNull; + +public class ProxyUtil { + private ProxyUtil() {} + + /** + * Wrap theInstance in a Proxy that synchronizes every method. + * + * @param theClass the target interface + * @param theInstance the instance to wrap + * @return a Proxy implementing theClass interface that syncronizes every call on theInstance + * @param the interface type + */ + public static T synchronizedProxy(Class theClass, T theInstance) { + Validate.isTrue(theClass.isInterface(), "%s is not an interface", theClass); + InvocationHandler handler = new SynchronizedHandler(theInstance); + Object object = Proxy.newProxyInstance(theClass.getClassLoader(), new Class[] { theClass }, handler); + return theClass.cast(object); + } + + /** + * Simple handler that first synchronizes on the delegate + */ + static class SynchronizedHandler implements InvocationHandler { + private final Object theDelegate; + + SynchronizedHandler(Object theDelegate) { + this.theDelegate = theDelegate; + } + + @Override + public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { + synchronized (theDelegate) { + return method.invoke(theDelegate, args); + } + } + } +} diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4622-batch2-chunk-io.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4622-batch2-chunk-io.yaml new file mode 100644 index 000000000000..7676575795b5 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_4_0/4622-batch2-chunk-io.yaml @@ -0,0 +1,4 @@ +--- +type: perf +issue: 4622 +title: "The batch system now reads less data during the maintenance pass. This avoids slowdowns on large systems." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4621-work-chunk-events.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4621-work-chunk-events.yaml new file mode 100644 index 000000000000..4535bcfd35fd --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4621-work-chunk-events.yaml @@ -0,0 +1,4 @@ +--- +type: change +issue: 4621 +title: "Batch2 work-chunk processing now aligns transaction boundaries with event transitions." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4643-fix-intermittent-in-batch2-tests.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4643-fix-intermittent-in-batch2-tests.yaml new file mode 100644 index 000000000000..293cf6962759 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4643-fix-intermittent-in-batch2-tests.yaml @@ -0,0 +1,5 @@ +--- +type: fix +issue: 4643 +title: "There was a transaction boundary issue in the Batch2 storage layer which resulted in the + framework needing more open database connections than necessary. This has been corrected." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4647-job-instance-transactions.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4647-job-instance-transactions.yaml new file mode 100644 index 000000000000..887d6068a082 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4647-job-instance-transactions.yaml @@ -0,0 +1,4 @@ +--- +type: fix +issue: 4647 +title: "Batch job state transitions are are now transitionally safe." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-bulk-export-resource-metadata.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-bulk-export-resource-metadata.yaml new file mode 100644 index 000000000000..d3ddc3784d9e --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-bulk-export-resource-metadata.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 4774 +title: "Bulk Export now supports a new `_exportId` parameter. If provided, any Binary resources generated by this export will have an extension in their `binary.meta` field which identifies this export. This can be used to correlate exported resources with the export job that generated them. +In addition, the `binary.meta` field of Bulk Export-generated binaries will also contain the job ID of the export job that generated them, as well as the resource type of the data contained within the binary." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-interceptor-for-prefixing.yaml b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-interceptor-for-prefixing.yaml new file mode 100644 index 000000000000..bb200f1a0592 --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4774-interceptor-for-prefixing.yaml @@ -0,0 +1,5 @@ +--- +type: add +issue: 4774 +title: "A new Pointcut called `STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX` has been added. This pointcut is called when a binary blob is about to be stored, +and allows implementers to attach a prefix to the blob ID before it is stored." diff --git a/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md new file mode 100644 index 000000000000..a5ced6d3945c --- /dev/null +++ b/hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md @@ -0,0 +1,68 @@ + +```mermaid +--- +title: Batch2 Job Instance state transitions +--- +stateDiagram-v2 + [*] --> QUEUED : on db create and queued on kakfa + QUEUED --> IN_PROGRESS : on any work-chunk received by worker + %% and (see ca.uhn.fhir.batch2.progress.InstanceProgress.getNewStatus()) + state first_step_finished <> + IN_PROGRESS --> first_step_finished : When 1st step finishes + first_step_finished --> COMPLETED: if no chunks produced + first_step_finished --> IN_PROGRESS: chunks produced + IN_PROGRESS --> in_progress_poll : on poll \n(count acomplete/failed/errored chunks) + in_progress_poll --> COMPLETED : 0 failures, errored, or incomplete\n AND at least 1 chunk complete + in_progress_poll --> ERRORED : no failed but errored chunks + in_progress_poll --> FINALIZE : none failed, gated execution\n last step\n queue REDUCER chunk + in_progress_poll --> IN_PROGRESS : still work to do + %% ERRORED is just like IN_PROGRESS, but it is a one-way trip from IN_PROGRESS to ERRORED. + %% FIXME We could probably delete/merge this state with IS_PROCESS, and use the error count in the UI. + note left of ERRORED + Parallel to IS_PROCESS + end note + state in_progress_poll <> + state error_progress_poll <> + ERRORED --> error_progress_poll : on poll \n(count acomplete/failed/errored chunks) + error_progress_poll --> FAILED : any failed chunks + error_progress_poll --> ERRORED : no failed but errored chunks + error_progress_poll --> FINALIZE : none failed, gated execution\n last step\n queue REDUCER chunk + error_progress_poll --> COMPLETED : 0 failures, errored, or incomplete AND at least 1 chunk complete + state do_report <> + FINALIZE --> do_reduction: poll util worker marks REDUCER chunk yes or no. + do_reduction --> COMPLETED : success + do_reduction --> FAILED : fail + in_progress_poll --> FAILED : any failed chunks +``` + +```mermaid +--- +title: Batch2 Job Work Chunk state transitions +--- +stateDiagram-v2 + state QUEUED + state on_receive <> + state IN_PROGRESS + state ERROR + state execute <> + state FAILED + state COMPLETED + direction LR + [*] --> QUEUED : on create + + %% worker processing states + QUEUED --> on_receive : on deque by worker + on_receive --> IN_PROGRESS : start execution + + IN_PROGRESS --> execute: execute + execute --> ERROR : on re-triable error + execute --> COMPLETED : success\n maybe trigger instance first_step_finished + execute --> FAILED : on unrecoverable \n or too many errors + + %% temporary error state until retry + ERROR --> on_receive : exception rollback\n triggers redelivery + + %% terminal states + COMPLETED --> [*] + FAILED --> [*] +``` diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/JobInstanceUtil.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java similarity index 61% rename from hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/JobInstanceUtil.java rename to hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java index 29fdc05800a1..723b9256b4a6 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/util/JobInstanceUtil.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtil.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.util; - /*- * #%L * HAPI FHIR JPA Server @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.WorkChunk; @@ -27,7 +26,7 @@ import javax.annotation.Nonnull; -public class JobInstanceUtil { +class JobInstanceUtil { private JobInstanceUtil() {} @@ -64,14 +63,44 @@ public static JobInstance fromEntityToInstance(@Nonnull Batch2JobInstanceEntity return retVal; } + /** + * Copies all JobInstance fields to a Batch2JobInstanceEntity + * @param theJobInstance the job + * @param theJobInstanceEntity the target entity + */ + public static void fromInstanceToEntity(@Nonnull JobInstance theJobInstance, @Nonnull Batch2JobInstanceEntity theJobInstanceEntity) { + theJobInstanceEntity.setId(theJobInstance.getInstanceId()); + theJobInstanceEntity.setDefinitionId(theJobInstance.getJobDefinitionId()); + theJobInstanceEntity.setDefinitionVersion(theJobInstance.getJobDefinitionVersion()); + theJobInstanceEntity.setStatus(theJobInstance.getStatus()); + theJobInstanceEntity.setCancelled(theJobInstance.isCancelled()); + theJobInstanceEntity.setFastTracking(theJobInstance.isFastTracking()); + theJobInstanceEntity.setStartTime(theJobInstance.getStartTime()); + theJobInstanceEntity.setCreateTime(theJobInstance.getCreateTime()); + theJobInstanceEntity.setEndTime(theJobInstance.getEndTime()); + theJobInstanceEntity.setUpdateTime(theJobInstance.getUpdateTime()); + theJobInstanceEntity.setCombinedRecordsProcessed(theJobInstance.getCombinedRecordsProcessed()); + theJobInstanceEntity.setCombinedRecordsProcessedPerSecond(theJobInstance.getCombinedRecordsProcessedPerSecond()); + theJobInstanceEntity.setTotalElapsedMillis(theJobInstance.getTotalElapsedMillis()); + theJobInstanceEntity.setWorkChunksPurged(theJobInstance.isWorkChunksPurged()); + theJobInstanceEntity.setProgress(theJobInstance.getProgress()); + theJobInstanceEntity.setErrorMessage(theJobInstance.getErrorMessage()); + theJobInstanceEntity.setErrorCount(theJobInstance.getErrorCount()); + theJobInstanceEntity.setEstimatedTimeRemaining(theJobInstance.getEstimatedTimeRemaining()); + theJobInstanceEntity.setParams(theJobInstance.getParameters()); + theJobInstanceEntity.setCurrentGatedStepId(theJobInstance.getCurrentGatedStepId()); + theJobInstanceEntity.setReport(theJobInstance.getReport()); + theJobInstanceEntity.setEstimatedTimeRemaining(theJobInstance.getEstimatedTimeRemaining()); + } + /** * Converts a Batch2WorkChunkEntity into a WorkChunk object + * * @param theEntity - the entity to convert - * @param theIncludeData - whether or not to include the Data attached to the chunk * @return - the WorkChunk object */ @Nonnull - public static WorkChunk fromEntityToWorkChunk(@Nonnull Batch2WorkChunkEntity theEntity, boolean theIncludeData) { + public static WorkChunk fromEntityToWorkChunk(@Nonnull Batch2WorkChunkEntity theEntity) { WorkChunk retVal = new WorkChunk(); retVal.setId(theEntity.getId()); retVal.setSequence(theEntity.getSequence()); @@ -87,11 +116,8 @@ public static WorkChunk fromEntityToWorkChunk(@Nonnull Batch2WorkChunkEntity the retVal.setErrorMessage(theEntity.getErrorMessage()); retVal.setErrorCount(theEntity.getErrorCount()); retVal.setRecordsProcessed(theEntity.getRecordsProcessed()); - if (theIncludeData) { - if (theEntity.getSerializedData() != null) { - retVal.setData(theEntity.getSerializedData()); - } - } + // note: may be null out if queried NoData + retVal.setData(theEntity.getSerializedData()); return retVal; } } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java index de3ef4a12b41..8951920a26e3 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.batch2; - /*- * #%L * HAPI FHIR JPA Server @@ -19,20 +17,24 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.config.BaseBatch2Config; -import ca.uhn.fhir.batch2.coordinator.SynchronizedJobPersistenceWrapper; import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; +import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.system.HapiSystemProperties; +import ca.uhn.fhir.util.ProxyUtil; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; import org.springframework.transaction.PlatformTransactionManager; +import javax.persistence.EntityManager; + @Configuration @Import({ BulkExportJobConfig.class @@ -40,19 +42,22 @@ public class JpaBatch2Config extends BaseBatch2Config { @Bean - public IJobPersistence batch2JobInstancePersister(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager) { - return new JpaJobPersistenceImpl(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager); + public IJobPersistence batch2JobInstancePersister(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, IHapiTransactionService theTransactionService, EntityManager theEntityManager) { + return new JpaJobPersistenceImpl(theJobInstanceRepository, theWorkChunkRepository, theTransactionService, theEntityManager); } @Primary @Bean - public IJobPersistence batch2JobInstancePersisterWrapper(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager) { - IJobPersistence retVal = batch2JobInstancePersister(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager); + public IJobPersistence batch2JobInstancePersisterWrapper(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, IHapiTransactionService theTransactionService, EntityManager theEntityManager) { + IJobPersistence retVal = batch2JobInstancePersister(theJobInstanceRepository, theWorkChunkRepository, theTransactionService, theEntityManager); // Avoid H2 synchronization issues caused by // https://github.com/h2database/h2database/issues/1808 - if (HapiSystemProperties.isUnitTestModeEnabled()) { - retVal = new SynchronizedJobPersistenceWrapper(retVal); - } + // TODO: Update 2023-03-14 - The bug above appears to be fixed. I'm going to try + // disabing this and see if we can get away without it. If so, we can delete + // this entirely +// if (HapiSystemProperties.isUnitTestModeEnabled()) { +// retVal = ProxyUtil.synchronizedProxy(IJobPersistence.class, retVal); +// } return retVal; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java index b2994b4026b1..7d3f0d9bb8ed 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImpl.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.batch2; - /*- * #%L * HAPI FHIR JPA Server @@ -19,21 +17,24 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobOperationResultJson; -import ca.uhn.fhir.batch2.coordinator.BatchWorkChunk; import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; import ca.uhn.fhir.batch2.model.JobInstance; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; +import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; -import ca.uhn.fhir.jpa.util.JobInstanceUtil; import ca.uhn.fhir.model.api.PagingIterator; import ca.uhn.fhir.util.Logs; import org.apache.commons.collections4.ListUtils; @@ -43,14 +44,15 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Pageable; import org.springframework.data.domain.Sort; -import org.springframework.transaction.PlatformTransactionManager; -import org.springframework.transaction.TransactionDefinition; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; import org.springframework.transaction.support.TransactionSynchronizationManager; -import org.springframework.transaction.support.TransactionTemplate; import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import javax.persistence.EntityManager; +import javax.persistence.LockModeType; +import javax.persistence.Query; import java.util.ArrayList; import java.util.Date; import java.util.Iterator; @@ -62,33 +64,33 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static ca.uhn.fhir.batch2.coordinator.WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT; import static ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity.ERROR_MSG_MAX_LENGTH; import static org.apache.commons.lang3.StringUtils.isBlank; public class JpaJobPersistenceImpl implements IJobPersistence { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); + public static final String CREATE_TIME = "myCreateTime"; private final IBatch2JobInstanceRepository myJobInstanceRepository; private final IBatch2WorkChunkRepository myWorkChunkRepository; - private final TransactionTemplate myTxTemplate; + private final EntityManager myEntityManager; + private final IHapiTransactionService myTransactionService; /** * Constructor */ - public JpaJobPersistenceImpl(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager) { + public JpaJobPersistenceImpl(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, IHapiTransactionService theTransactionService, EntityManager theEntityManager) { Validate.notNull(theJobInstanceRepository); Validate.notNull(theWorkChunkRepository); myJobInstanceRepository = theJobInstanceRepository; myWorkChunkRepository = theWorkChunkRepository; - - // TODO: JA replace with HapiTransactionManager in megascale ticket - myTxTemplate = new TransactionTemplate(theTransactionManager); - myTxTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + myTransactionService = theTransactionService; + myEntityManager = theEntityManager; } @Override - @Transactional(propagation = Propagation.REQUIRED) - public String storeWorkChunk(BatchWorkChunk theBatchWorkChunk) { + public String onWorkChunkCreate(WorkChunkCreateEvent theBatchWorkChunk) { Batch2WorkChunkEntity entity = new Batch2WorkChunkEntity(); entity.setId(UUID.randomUUID().toString()); entity.setSequence(theBatchWorkChunk.sequence); @@ -99,21 +101,25 @@ public String storeWorkChunk(BatchWorkChunk theBatchWorkChunk) { entity.setSerializedData(theBatchWorkChunk.serializedData); entity.setCreateTime(new Date()); entity.setStartTime(new Date()); - entity.setStatus(StatusEnum.QUEUED); + entity.setStatus(WorkChunkStatusEnum.QUEUED); + ourLog.debug("Create work chunk {}/{}/{}", entity.getInstanceId(), entity.getId(), entity.getTargetStepId()); + ourLog.trace("Create work chunk data {}/{}: {}", entity.getInstanceId(), entity.getId(), entity.getSerializedData()); myWorkChunkRepository.save(entity); return entity.getId(); } @Override @Transactional(propagation = Propagation.REQUIRED) - public Optional fetchWorkChunkSetStartTimeAndMarkInProgress(String theChunkId) { - int rowsModified = myWorkChunkRepository.updateChunkStatusForStart(theChunkId, new Date(), StatusEnum.IN_PROGRESS, List.of(StatusEnum.QUEUED, StatusEnum.ERRORED, StatusEnum.IN_PROGRESS)); + public Optional onWorkChunkDequeue(String theChunkId) { + // NOTE: Ideally, IN_PROGRESS wouldn't be allowed here. On chunk failure, we probably shouldn't be allowed. But how does re-run happen if k8s kills a processor mid run? + List priorStates = List.of(WorkChunkStatusEnum.QUEUED, WorkChunkStatusEnum.ERRORED, WorkChunkStatusEnum.IN_PROGRESS); + int rowsModified = myWorkChunkRepository.updateChunkStatusForStart(theChunkId, new Date(), WorkChunkStatusEnum.IN_PROGRESS, priorStates); if (rowsModified == 0) { ourLog.info("Attempting to start chunk {} but it was already started.", theChunkId); return Optional.empty(); } else { Optional chunk = myWorkChunkRepository.findById(theChunkId); - return chunk.map(t -> toChunk(t, true)); + return chunk.map(this::toChunk); } } @@ -147,14 +153,14 @@ public List fetchInstances(String theJobDefinitionId, Set fetchInstancesByJobDefinitionIdAndStatus(String theJobDefinitionId, Set theRequestedStatuses, int thePageSize, int thePageIndex) { - PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, "myCreateTime"); + PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, CREATE_TIME); return toInstanceList(myJobInstanceRepository.fetchInstancesByJobDefinitionIdAndStatus(theJobDefinitionId, theRequestedStatuses, pageRequest)); } @Override @Transactional(propagation = Propagation.REQUIRES_NEW) public List fetchInstancesByJobDefinitionId(String theJobDefinitionId, int thePageSize, int thePageIndex) { - PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, "myCreateTime"); + PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, CREATE_TIME); return toInstanceList(myJobInstanceRepository.findInstancesByJobDefinitionId(theJobDefinitionId, pageRequest)); } @@ -178,9 +184,10 @@ private List toInstanceList(List theInstan @Override @Nonnull - @Transactional(propagation = Propagation.REQUIRES_NEW) public Optional fetchInstance(String theInstanceId) { - return myJobInstanceRepository.findById(theInstanceId).map(this::toInstance); + return myTransactionService + .withRequest(null) + .execute(() -> myJobInstanceRepository.findById(theInstanceId).map(this::toInstance)); } @Override @@ -215,19 +222,19 @@ public List fetchInstances(FetchJobInstancesRequest theRequest, int @Transactional(propagation = Propagation.REQUIRES_NEW) public List fetchInstances(int thePageSize, int thePageIndex) { // default sort is myCreateTime Asc - PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, "myCreateTime"); - return myJobInstanceRepository.findAll(pageRequest).stream().map(t -> toInstance(t)).collect(Collectors.toList()); + PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.ASC, CREATE_TIME); + return myJobInstanceRepository.findAll(pageRequest).stream().map(this::toInstance).collect(Collectors.toList()); } @Override @Transactional(propagation = Propagation.REQUIRES_NEW) public List fetchRecentInstances(int thePageSize, int thePageIndex) { - PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.DESC, "myCreateTime"); + PageRequest pageRequest = PageRequest.of(thePageIndex, thePageSize, Sort.Direction.DESC, CREATE_TIME); return myJobInstanceRepository.findAll(pageRequest).stream().map(this::toInstance).collect(Collectors.toList()); } - private WorkChunk toChunk(Batch2WorkChunkEntity theEntity, boolean theIncludeData) { - return JobInstanceUtil.fromEntityToWorkChunk(theEntity, theIncludeData); + private WorkChunk toChunk(Batch2WorkChunkEntity theEntity) { + return JobInstanceUtil.fromEntityToWorkChunk(theEntity); } private JobInstance toInstance(Batch2JobInstanceEntity theEntity) { @@ -236,29 +243,44 @@ private JobInstance toInstance(Batch2JobInstanceEntity theEntity) { @Override @Transactional(propagation = Propagation.REQUIRES_NEW) - public void markWorkChunkAsErroredAndIncrementErrorCount(String theChunkId, String theErrorMessage) { - String errorMessage = truncateErrorMessage(theErrorMessage); - myWorkChunkRepository.updateChunkStatusAndIncrementErrorCountForEndError(theChunkId, new Date(), errorMessage, StatusEnum.ERRORED); + public WorkChunkStatusEnum onWorkChunkError(WorkChunkErrorEvent theParameters) { + String chunkId = theParameters.getChunkId(); + String errorMessage = truncateErrorMessage(theParameters.getErrorMsg()); + int changeCount = myWorkChunkRepository.updateChunkStatusAndIncrementErrorCountForEndError(chunkId, new Date(), errorMessage, WorkChunkStatusEnum.ERRORED); + Validate.isTrue(changeCount>0, "changed chunk matching %s", chunkId); + + Query query = myEntityManager.createQuery( + "update Batch2WorkChunkEntity " + + "set myStatus = :failed " + + ",myErrorMessage = CONCAT('Too many errors: ', myErrorCount, '. Last error msg was ', myErrorMessage) " + + "where myId = :chunkId and myErrorCount > :maxCount"); + query.setParameter("chunkId", chunkId); + query.setParameter("failed", WorkChunkStatusEnum.FAILED); + query.setParameter("maxCount", MAX_CHUNK_ERROR_COUNT); + int failChangeCount = query.executeUpdate(); + + if (failChangeCount > 0) { + return WorkChunkStatusEnum.FAILED; + } else { + return WorkChunkStatusEnum.ERRORED; + } } @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWorkChunkAsErrorRequest theParameters) { - markWorkChunkAsErroredAndIncrementErrorCount(theParameters.getChunkId(), theParameters.getErrorMsg()); - Optional op = myWorkChunkRepository.findById(theParameters.getChunkId()); - - return op.map(c -> toChunk(c, theParameters.isIncludeData())); + @Transactional + public void onWorkChunkFailed(String theChunkId, String theErrorMessage) { + ourLog.info("Marking chunk {} as failed with message: {}", theChunkId, theErrorMessage); + String errorMessage = truncateErrorMessage(theErrorMessage); + myWorkChunkRepository.updateChunkStatusAndIncrementErrorCountForEndError(theChunkId, new Date(), errorMessage, WorkChunkStatusEnum.FAILED); } @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void markWorkChunkAsFailed(String theChunkId, String theErrorMessage) { - ourLog.info("Marking chunk {} as failed with message: {}", theChunkId, theErrorMessage); - String errorMessage = truncateErrorMessage(theErrorMessage); - myWorkChunkRepository.updateChunkStatusAndIncrementErrorCountForEndError(theChunkId, new Date(), errorMessage, StatusEnum.FAILED); + @Transactional + public void onWorkChunkCompletion(WorkChunkCompletionEvent theEvent) { + myWorkChunkRepository.updateChunkStatusAndClearDataForEndSuccess(theEvent.getChunkId(), new Date(), theEvent.getRecordsProcessed(), theEvent.getRecoveredErrorCount(), WorkChunkStatusEnum.COMPLETED); } - @Nonnull + @Nullable private static String truncateErrorMessage(String theErrorMessage) { String errorMessage; if (theErrorMessage != null && theErrorMessage.length() > ERROR_MSG_MAX_LENGTH) { @@ -271,15 +293,7 @@ private static String truncateErrorMessage(String theErrorMessage) { } @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public void markWorkChunkAsCompletedAndClearData(String theInstanceId, String theChunkId, int theRecordsProcessed) { - StatusEnum newStatus = StatusEnum.COMPLETED; - ourLog.debug("Marking chunk {} for instance {} to status {}", theChunkId, theInstanceId, newStatus); - myWorkChunkRepository.updateChunkStatusAndClearDataForEndSuccess(theChunkId, new Date(), theRecordsProcessed, newStatus); - } - - @Override - public void markWorkChunksWithStatusAndWipeData(String theInstanceId, List theChunkIds, StatusEnum theStatus, String theErrorMessage) { + public void markWorkChunksWithStatusAndWipeData(String theInstanceId, List theChunkIds, WorkChunkStatusEnum theStatus, String theErrorMessage) { assert TransactionSynchronizationManager.isActualTransactionActive(); ourLog.debug("Marking all chunks for instance {} to status {}", theInstanceId, theStatus); @@ -290,25 +304,20 @@ public void markWorkChunksWithStatusAndWipeData(String theInstanceId, List instance = myJobInstanceRepository.findById(theInstanceId); - if (!instance.isPresent()) { + if (instance.isEmpty()) { return false; } if (instance.get().getStatus().isEnded()) { return false; } - List statusesForStep = myWorkChunkRepository.getDistinctStatusesForStep(theInstanceId, theCurrentStepId); + Set statusesForStep = myWorkChunkRepository.getDistinctStatusesForStep(theInstanceId, theCurrentStepId); + ourLog.debug("Checking whether gated job can advanced to next step. [instanceId={}, currentStepId={}, statusesForStep={}]", theInstanceId, theCurrentStepId, statusesForStep); - return statusesForStep.stream().noneMatch(StatusEnum::isIncomplete) && statusesForStep.stream().anyMatch(status -> status == StatusEnum.COMPLETED); + return statusesForStep.isEmpty() || statusesForStep.equals(Set.of(WorkChunkStatusEnum.COMPLETED)); } /** @@ -322,29 +331,28 @@ public List fetchWorkChunksWithoutData(String theInstanceId, int theP } private void fetchChunks(String theInstanceId, boolean theIncludeData, int thePageSize, int thePageIndex, Consumer theConsumer) { - if (theIncludeData) { - // I think this is dead: MB - myTxTemplate.executeWithoutResult(tx -> { - List chunks = myWorkChunkRepository.fetchChunks(PageRequest.of(thePageIndex, thePageSize), theInstanceId); - for (Batch2WorkChunkEntity chunk : chunks) { - theConsumer.accept(toChunk(chunk, theIncludeData)); + myTransactionService + .withRequest(null) + .withPropagation(Propagation.REQUIRES_NEW) + .execute(() -> { + List chunks; + if (theIncludeData) { + chunks = myWorkChunkRepository.fetchChunks(PageRequest.of(thePageIndex, thePageSize), theInstanceId); + } else { + chunks = myWorkChunkRepository.fetchChunksNoData(PageRequest.of(thePageIndex, thePageSize), theInstanceId); } - }); - } else { - // wipmb mb here - // a minimally-different path for a prod-fix. - myTxTemplate.executeWithoutResult(tx -> { - List chunks = myWorkChunkRepository.fetchChunksNoData(PageRequest.of(thePageIndex, thePageSize), theInstanceId); for (Batch2WorkChunkEntity chunk : chunks) { - theConsumer.accept(toChunk(chunk, theIncludeData)); + theConsumer.accept(toChunk(chunk)); } }); - } } @Override - public List fetchallchunkidsforstepWithStatus(String theInstanceId, String theStepId, StatusEnum theStatusEnum) { - return myTxTemplate.execute(tx -> myWorkChunkRepository.fetchAllChunkIdsForStepWithStatus(theInstanceId, theStepId, theStatusEnum)); + public List fetchAllChunkIdsForStepWithStatus(String theInstanceId, String theStepId, WorkChunkStatusEnum theStatusEnum) { + return myTransactionService + .withRequest(null) + .withPropagation(Propagation.REQUIRES_NEW) + .execute(() -> myWorkChunkRepository.fetchAllChunkIdsForStepWithStatus(theInstanceId, theStepId, theStatusEnum)); } @Override @@ -352,75 +360,40 @@ public void updateInstanceUpdateTime(String theInstanceId) { myJobInstanceRepository.updateInstanceUpdateTime(theInstanceId, new Date()); } - private void fetchChunksForStep(String theInstanceId, String theStepId, int thePageSize, int thePageIndex, Consumer theConsumer) { - myTxTemplate.executeWithoutResult(tx -> { - List chunks = myWorkChunkRepository.fetchChunksForStep(PageRequest.of(thePageIndex, thePageSize), theInstanceId, theStepId); - for (Batch2WorkChunkEntity chunk : chunks) { - theConsumer.accept(toChunk(chunk, true)); - } - }); - } - /** * Note: Not @Transactional because the transaction happens in a lambda that's called outside of this method's scope */ @Override public Iterator fetchAllWorkChunksIterator(String theInstanceId, boolean theWithData) { - // wipmb mb here return new PagingIterator<>((thePageIndex, theBatchSize, theConsumer) -> fetchChunks(theInstanceId, theWithData, theBatchSize, thePageIndex, theConsumer)); } - /** - * Deprecated, use {@link ca.uhn.fhir.jpa.batch2.JpaJobPersistenceImpl#fetchAllWorkChunksForStepStream(String, String)} - * Note: Not @Transactional because the transaction happens in a lambda that's called outside of this method's scope - */ @Override - @Deprecated - public Iterator fetchAllWorkChunksForStepIterator(String theInstanceId, String theStepId) { - return new PagingIterator<>((thePageIndex, theBatchSize, theConsumer) -> fetchChunksForStep(theInstanceId, theStepId, theBatchSize, thePageIndex, theConsumer)); - } - - @Override - @Transactional(propagation = Propagation.MANDATORY) public Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId) { - return myWorkChunkRepository.fetchChunksForStep(theInstanceId, theStepId).map((entity) -> toChunk(entity, true)); + return myWorkChunkRepository.fetchChunksForStep(theInstanceId, theStepId).map(this::toChunk); } - /** - * Update the stored instance - * - * @param theInstance The instance - Must contain an ID - * @return true if the status changed - */ @Override - @Transactional(propagation = Propagation.REQUIRES_NEW) - public boolean updateInstance(JobInstance theInstance) { - // Separate updating the status so we have atomic information about whether the status is changing - int recordsChangedByStatusUpdate = myJobInstanceRepository.updateInstanceStatus(theInstance.getInstanceId(), theInstance.getStatus()); - - Optional instanceOpt = myJobInstanceRepository.findById(theInstance.getInstanceId()); - Batch2JobInstanceEntity instanceEntity = instanceOpt.orElseThrow(() -> new IllegalArgumentException("Unknown instance ID: " + theInstance.getInstanceId())); - - instanceEntity.setStartTime(theInstance.getStartTime()); - instanceEntity.setEndTime(theInstance.getEndTime()); - instanceEntity.setStatus(theInstance.getStatus()); - instanceEntity.setCancelled(theInstance.isCancelled()); - instanceEntity.setFastTracking(theInstance.isFastTracking()); - instanceEntity.setCombinedRecordsProcessed(theInstance.getCombinedRecordsProcessed()); - instanceEntity.setCombinedRecordsProcessedPerSecond(theInstance.getCombinedRecordsProcessedPerSecond()); - instanceEntity.setTotalElapsedMillis(theInstance.getTotalElapsedMillis()); - instanceEntity.setWorkChunksPurged(theInstance.isWorkChunksPurged()); - instanceEntity.setProgress(theInstance.getProgress()); - instanceEntity.setErrorMessage(theInstance.getErrorMessage()); - instanceEntity.setErrorCount(theInstance.getErrorCount()); - instanceEntity.setEstimatedTimeRemaining(theInstance.getEstimatedTimeRemaining()); - instanceEntity.setCurrentGatedStepId(theInstance.getCurrentGatedStepId()); - instanceEntity.setReport(theInstance.getReport()); - - myJobInstanceRepository.save(instanceEntity); - - return recordsChangedByStatusUpdate > 0; + @Transactional + public boolean updateInstance(String theInstanceId, JobInstanceUpdateCallback theModifier) { + Batch2JobInstanceEntity instanceEntity = myEntityManager.find(Batch2JobInstanceEntity.class, theInstanceId, LockModeType.PESSIMISTIC_WRITE); + if (null == instanceEntity) { + ourLog.error("No instance found with Id {}", theInstanceId); + return false; + } + // convert to JobInstance for public api + JobInstance jobInstance = JobInstanceUtil.fromEntityToInstance(instanceEntity); + + // run the modification callback + boolean wasModified = theModifier.doUpdate(jobInstance); + + if (wasModified) { + // copy fields back for flush. + JobInstanceUtil.fromInstanceToEntity(jobInstance, instanceEntity); + } + + return wasModified; } @Override @@ -448,7 +421,16 @@ public boolean markInstanceAsCompleted(String theInstanceId) { @Override public boolean markInstanceAsStatus(String theInstance, StatusEnum theStatusEnum) { + int recordsChanged = myTransactionService + .withRequest(null) + .execute(()->myJobInstanceRepository.updateInstanceStatus(theInstance, theStatusEnum)); + return recordsChanged > 0; + } + + @Override + public boolean markInstanceAsStatusWhenStatusIn(String theInstance, StatusEnum theStatusEnum, Set thePriorStates) { int recordsChanged = myJobInstanceRepository.updateInstanceStatus(theInstance, theStatusEnum); + ourLog.debug("Update job {} to status {} if in status {}: {}", theInstance, theStatusEnum, thePriorStates, recordsChanged>0); return recordsChanged > 0; } @@ -458,15 +440,34 @@ public JobOperationResultJson cancelInstance(String theInstanceId) { int recordsChanged = myJobInstanceRepository.updateInstanceCancelled(theInstanceId, true); String operationString = "Cancel job instance " + theInstanceId; + // TODO MB this is much too detailed to be down here - this should be up at the api layer. Replace with simple enum. + String messagePrefix = "Job instance <" + theInstanceId + ">"; if (recordsChanged > 0) { - return JobOperationResultJson.newSuccess(operationString, "Job instance <" + theInstanceId + "> successfully cancelled."); + return JobOperationResultJson.newSuccess(operationString, messagePrefix + " successfully cancelled."); } else { Optional instance = fetchInstance(theInstanceId); if (instance.isPresent()) { - return JobOperationResultJson.newFailure(operationString, "Job instance <" + theInstanceId + "> was already cancelled. Nothing to do."); + return JobOperationResultJson.newFailure(operationString, messagePrefix + " was already cancelled. Nothing to do."); } else { - return JobOperationResultJson.newFailure(operationString, "Job instance <" + theInstanceId + "> not found."); + return JobOperationResultJson.newFailure(operationString, messagePrefix + " not found."); } } } + + + @Override + public void processCancelRequests() { + myTransactionService + .withSystemRequest() + .execute(()->{ + Query query = myEntityManager.createQuery( + "UPDATE Batch2JobInstanceEntity b " + + "set myStatus = ca.uhn.fhir.batch2.model.StatusEnum.CANCELLED " + + "where myCancelled = true " + + "AND myStatus IN (:states)"); + query.setParameter("states", StatusEnum.CANCELLED.getPriorStates()); + query.executeUpdate(); + }); + } + } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java index 1bd900b4fe67..8aa9a032c648 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/BaseHapiFhirResourceDao.java @@ -29,6 +29,7 @@ import ca.uhn.fhir.context.RuntimeResourceDefinition; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.interceptor.model.ReadPartitionIdRequestDetails; import ca.uhn.fhir.interceptor.model.RequestPartitionId; @@ -161,6 +162,8 @@ public abstract class BaseHapiFhirResourceDao extends B public static final String BASE_RESOURCE_NAME = "resource"; private static final org.slf4j.Logger ourLog = org.slf4j.LoggerFactory.getLogger(BaseHapiFhirResourceDao.class); + @Autowired + protected IInterceptorBroadcaster myInterceptorBroadcaster; @Autowired protected PlatformTransactionManager myPlatformTransactionManager; diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java index bc8e24e09287..d0fc3d6bac3d 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2JobInstanceRepository.java @@ -38,6 +38,10 @@ public interface IBatch2JobInstanceRepository extends JpaRepository :status") int updateInstanceStatus(@Param("id") String theInstanceId, @Param("status") StatusEnum theStatus); + @Modifying + @Query("UPDATE Batch2JobInstanceEntity e SET e.myStatus = :status WHERE e.myId = :id and e.myStatus IN ( :prior_states )") + int updateInstanceStatusIfIn(@Param("id") String theInstanceId, @Param("status") StatusEnum theNewState, @Param("prior_states") Set thePriorStates); + @Modifying @Query("UPDATE Batch2JobInstanceEntity e SET e.myUpdateTime = :updated WHERE e.myId = :id") int updateInstanceUpdateTime(@Param("id") String theInstanceId, @Param("updated") Date theUpdated); diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java index d703da4461e4..25729c205a93 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/dao/data/IBatch2WorkChunkRepository.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.dao.data; - /*- * #%L * HAPI FHIR JPA Server @@ -19,8 +17,9 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.dao.data; -import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; import org.springframework.data.domain.Pageable; import org.springframework.data.jpa.repository.JpaRepository; @@ -28,13 +27,17 @@ import org.springframework.data.jpa.repository.Query; import org.springframework.data.repository.query.Param; +import java.util.Collection; import java.util.Date; import java.util.List; +import java.util.Set; import java.util.stream.Stream; public interface IBatch2WorkChunkRepository extends JpaRepository, IHapiFhirJpaRepository { - @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId ORDER BY e.mySequence ASC") + // NOTE we need a stable sort so paging is reliable. + // Warning: mySequence is not unique - it is reset for every chunk. So we also sort by myId. + @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId ORDER BY e.mySequence ASC, e.myId ASC") List fetchChunks(Pageable thePageRequest, @Param("instanceId") String theInstanceId); /** @@ -45,49 +48,38 @@ public interface IBatch2WorkChunkRepository extends JpaRepository fetchChunksNoData(Pageable thePageRequest, @Param("instanceId") String theInstanceId); @Query("SELECT DISTINCT e.myStatus from Batch2WorkChunkEntity e where e.myInstanceId = :instanceId AND e.myTargetStepId = :stepId") - List getDistinctStatusesForStep(@Param("instanceId") String theInstanceId, @Param("stepId") String theStepId); - - /** - * Deprecated, use {@link ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository#fetchChunksForStep(String, String)} - */ - @Deprecated - @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId AND e.myTargetStepId = :targetStepId ORDER BY e.mySequence ASC") - List fetchChunksForStep(Pageable thePageRequest, @Param("instanceId") String theInstanceId, @Param("targetStepId") String theTargetStepId); + Set getDistinctStatusesForStep(@Param("instanceId") String theInstanceId, @Param("stepId") String theStepId); @Query("SELECT e FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId AND e.myTargetStepId = :targetStepId ORDER BY e.mySequence ASC") Stream fetchChunksForStep(@Param("instanceId") String theInstanceId, @Param("targetStepId") String theTargetStepId); @Modifying - @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.myRecordsProcessed = :rp, e.mySerializedData = null WHERE e.myId = :id") - void updateChunkStatusAndClearDataForEndSuccess(@Param("id") String theChunkId, @Param("et") Date theEndTime, @Param("rp") int theRecordsProcessed, @Param("status") StatusEnum theInProgress); + @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, " + + "e.myRecordsProcessed = :rp, e.myErrorCount = e.myErrorCount + :errorRetries, e.mySerializedData = null " + + "WHERE e.myId = :id") + void updateChunkStatusAndClearDataForEndSuccess(@Param("id") String theChunkId, @Param("et") Date theEndTime, + @Param("rp") int theRecordsProcessed, @Param("errorRetries") int theErrorRetries, @Param("status") WorkChunkStatusEnum theInProgress); @Modifying @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.mySerializedData = null, e.myErrorMessage = :em WHERE e.myId IN(:ids)") - void updateAllChunksForInstanceStatusClearDataAndSetError(@Param("ids") List theChunkIds, @Param("et") Date theEndTime, @Param("status") StatusEnum theInProgress, @Param("em") String theError); + void updateAllChunksForInstanceStatusClearDataAndSetError(@Param("ids") List theChunkIds, @Param("et") Date theEndTime, @Param("status") WorkChunkStatusEnum theInProgress, @Param("em") String theError); @Modifying @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myEndTime = :et, e.myErrorMessage = :em, e.myErrorCount = e.myErrorCount + 1 WHERE e.myId = :id") - void updateChunkStatusAndIncrementErrorCountForEndError(@Param("id") String theChunkId, @Param("et") Date theEndTime, @Param("em") String theErrorMessage, @Param("status") StatusEnum theInProgress); + int updateChunkStatusAndIncrementErrorCountForEndError(@Param("id") String theChunkId, @Param("et") Date theEndTime, @Param("em") String theErrorMessage, @Param("status") WorkChunkStatusEnum theInProgress); @Modifying @Query("UPDATE Batch2WorkChunkEntity e SET e.myStatus = :status, e.myStartTime = :st WHERE e.myId = :id AND e.myStatus IN :startStatuses") - int updateChunkStatusForStart(@Param("id") String theChunkId, @Param("st") Date theStartedTime, @Param("status") StatusEnum theInProgress, @Param("startStatuses") List theStartStatuses); + int updateChunkStatusForStart(@Param("id") String theChunkId, @Param("st") Date theStartedTime, @Param("status") WorkChunkStatusEnum theInProgress, @Param("startStatuses") Collection theStartStatuses); @Modifying @Query("DELETE FROM Batch2WorkChunkEntity e WHERE e.myInstanceId = :instanceId") void deleteAllForInstance(@Param("instanceId") String theInstanceId); - @Modifying - @Query("UPDATE Batch2WorkChunkEntity e SET e.myErrorCount = e.myErrorCount + :by WHERE e.myId = :id") - void incrementWorkChunkErrorCount(@Param("id") String theChunkId, @Param("by") int theIncrementBy); - - @Query("SELECT e.myId from Batch2WorkChunkEntity e where e.myInstanceId = :instanceId AND e.myTargetStepId = :stepId") - List fetchAllChunkIdsForStep(@Param("instanceId") String theInstanceId, @Param("stepId") String theStepId); - @Query("SELECT e.myId from Batch2WorkChunkEntity e where e.myInstanceId = :instanceId AND e.myTargetStepId = :stepId AND e.myStatus = :status") - List fetchAllChunkIdsForStepWithStatus(@Param("instanceId")String theInstanceId, @Param("stepId")String theStepId, @Param("status")StatusEnum theStatus); + List fetchAllChunkIdsForStepWithStatus(@Param("instanceId")String theInstanceId, @Param("stepId")String theStepId, @Param("status") WorkChunkStatusEnum theStatus); } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java index 789a4267ef15..1a1803cc9233 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2JobInstanceEntity.java @@ -196,6 +196,10 @@ public void setEndTime(Date theEndTime) { myEndTime = theEndTime; } + public void setUpdateTime(Date theTime) { + myUpdateTime = theTime; + } + public Date getUpdateTime() { return myUpdateTime; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java index 73ddf421663b..894cddb032e2 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/entity/Batch2WorkChunkEntity.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.entity; - /*- * #%L * HAPI FHIR JPA Server @@ -19,8 +17,9 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.entity; -import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import org.apache.commons.lang3.builder.ToStringBuilder; import org.apache.commons.lang3.builder.ToStringStyle; @@ -87,7 +86,7 @@ public class Batch2WorkChunkEntity implements Serializable { private String mySerializedData; @Column(name = "STAT", length = STATUS_MAX_LENGTH, nullable = false) @Enumerated(EnumType.STRING) - private StatusEnum myStatus; + private WorkChunkStatusEnum myStatus; @ManyToOne(fetch = FetchType.LAZY) @JoinColumn(name = "INSTANCE_ID", insertable = false, updatable = false, foreignKey = @ForeignKey(name = "FK_BT2WC_INSTANCE")) private Batch2JobInstanceEntity myInstance; @@ -109,7 +108,7 @@ public Batch2WorkChunkEntity() { * Projection constructor for no-date path. */ public Batch2WorkChunkEntity(String theId, int theSequence, String theJobDefinitionId, int theJobDefinitionVersion, - String theInstanceId, String theTargetStepId, StatusEnum theStatus, + String theInstanceId, String theTargetStepId, WorkChunkStatusEnum theStatus, Date theCreateTime, Date theStartTime, Date theUpdateTime, Date theEndTime, String theErrorMessage, int theErrorCount, Integer theRecordsProcessed) { myId = theId; @@ -228,11 +227,11 @@ public void setSerializedData(String theSerializedData) { mySerializedData = theSerializedData; } - public StatusEnum getStatus() { + public WorkChunkStatusEnum getStatus() { return myStatus; } - public void setStatus(StatusEnum theStatus) { + public void setStatus(WorkChunkStatusEnum theStatus) { myStatus = theStatus; } diff --git a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java index 7784e22ceb64..00271a4dcfed 100644 --- a/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java +++ b/hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/search/builder/sql/SearchQueryExecutor.java @@ -121,7 +121,6 @@ private void fetchNext() { ourLog.trace("About to execute SQL: {}", sql); - // FIXME: update in ja_20230422_postgres_optimization hibernateQuery.setFetchSize(500000); hibernateQuery.setCacheable(false); hibernateQuery.setCacheMode(CacheMode.IGNORE); diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtilTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtilTest.java new file mode 100644 index 000000000000..ae0e69d65614 --- /dev/null +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JobInstanceUtilTest.java @@ -0,0 +1,30 @@ +package ca.uhn.fhir.jpa.batch2; + +import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; +import ca.uhn.fhir.test.utilities.RandomDataHelper; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class JobInstanceUtilTest { + + /** + * Fill with random data and round-trip via instance. + */ + @Test + void fromEntityToInstance() { + JobInstance instance = new JobInstance(); + RandomDataHelper.fillFieldsRandomly(instance); + + Batch2JobInstanceEntity entity = new Batch2JobInstanceEntity(); + JobInstanceUtil.fromInstanceToEntity(instance, entity); + JobInstance instanceCopyBack = JobInstanceUtil.fromEntityToInstance(entity); + + assertTrue(EqualsBuilder.reflectionEquals(instance, instanceCopyBack)); + + + } + +} diff --git a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java index b632e86b41ad..50c7ca4ff59b 100644 --- a/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java +++ b/hapi-fhir-jpaserver-base/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java @@ -6,17 +6,20 @@ import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; +import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; +import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; -import ca.uhn.fhir.jpa.util.JobInstanceUtil; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.InjectMocks; import org.mockito.Mock; +import org.mockito.Spy; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.data.domain.Pageable; -import org.springframework.transaction.PlatformTransactionManager; +import java.time.LocalDate; +import java.time.ZoneOffset; import java.util.Arrays; import java.util.Date; import java.util.List; @@ -25,9 +28,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -39,8 +40,8 @@ class JpaJobPersistenceImplTest { IBatch2JobInstanceRepository myJobInstanceRepository; @Mock IBatch2WorkChunkRepository myWorkChunkRepository; - @Mock - PlatformTransactionManager myTxManager; + @Spy + IHapiTransactionService myTxManager = new NonTransactionalHapiTransactionService(); @InjectMocks JpaJobPersistenceImpl mySvc; @@ -111,7 +112,7 @@ public void deleteChunks_withInstanceId_callsChunkRepoDelete() { // verify verify(myWorkChunkRepository) - .deleteAllForInstance(eq(jobId)); + .deleteAllForInstance(jobId); } @Test @@ -124,63 +125,9 @@ public void deleteInstanceAndChunks_withInstanceId_callsBothWorkchunkAndJobRespo // verify verify(myWorkChunkRepository) - .deleteAllForInstance(eq(jobid)); - verify(myJobInstanceRepository) - .deleteById(eq(jobid)); - } - - @Test - public void updateInstance_withInstance_checksInstanceExistsAndCallsSave() { - // setup - JobInstance toSave = createJobInstanceWithDemoData(); - Batch2JobInstanceEntity entity = new Batch2JobInstanceEntity(); - entity.setId(toSave.getInstanceId()); - - // when - when(myJobInstanceRepository.findById(eq(toSave.getInstanceId()))) - .thenReturn(Optional.of(entity)); - - // test - mySvc.updateInstance(toSave); - - // verify - ArgumentCaptor entityCaptor = ArgumentCaptor.forClass(Batch2JobInstanceEntity.class); + .deleteAllForInstance(jobid); verify(myJobInstanceRepository) - .save(entityCaptor.capture()); - Batch2JobInstanceEntity saved = entityCaptor.getValue(); - assertEquals(toSave.getInstanceId(), saved.getId()); - assertEquals(toSave.getStatus(), saved.getStatus()); - assertEquals(toSave.getStartTime(), entity.getStartTime()); - assertEquals(toSave.getEndTime(), entity.getEndTime()); - assertEquals(toSave.isCancelled(), entity.isCancelled()); - assertEquals(toSave.getCombinedRecordsProcessed(), entity.getCombinedRecordsProcessed()); - assertEquals(toSave.getCombinedRecordsProcessedPerSecond(), entity.getCombinedRecordsProcessedPerSecond()); - assertEquals(toSave.getTotalElapsedMillis(), entity.getTotalElapsedMillis()); - assertEquals(toSave.isWorkChunksPurged(), entity.getWorkChunksPurged()); - assertEquals(toSave.getProgress(), entity.getProgress()); - assertEquals(toSave.getErrorMessage(), entity.getErrorMessage()); - assertEquals(toSave.getErrorCount(), entity.getErrorCount()); - assertEquals(toSave.getEstimatedTimeRemaining(), entity.getEstimatedTimeRemaining()); - assertEquals(toSave.getCurrentGatedStepId(), entity.getCurrentGatedStepId()); - assertEquals(toSave.getReport(), entity.getReport()); - } - - @Test - public void updateInstance_invalidId_throwsIllegalArgumentException() { - // setup - JobInstance instance = createJobInstanceWithDemoData(); - - // when - when(myJobInstanceRepository.findById(anyString())) - .thenReturn(Optional.empty()); - - // test - try { - mySvc.updateInstance(instance); - fail(); - } catch (IllegalArgumentException ex) { - assertTrue(ex.getMessage().contains("Unknown instance ID: " + instance.getInstanceId())); - } + .deleteById(jobid); } @Test @@ -229,7 +176,7 @@ public void fetchInstance_validId_returnsInstance() { JobInstance instance = createJobInstanceFromEntity(entity); // when - when(myJobInstanceRepository.findById(eq(instance.getInstanceId()))) + when(myJobInstanceRepository.findById(instance.getInstanceId())) .thenReturn(Optional.of(entity)); // test @@ -240,10 +187,6 @@ public void fetchInstance_validId_returnsInstance() { assertEquals(instance.getInstanceId(), retInstance.get().getInstanceId()); } - private JobInstance createJobInstanceWithDemoData() { - return createJobInstanceFromEntity(createBatch2JobInstanceEntity()); - } - private JobInstance createJobInstanceFromEntity(Batch2JobInstanceEntity theEntity) { return JobInstanceUtil.fromEntityToInstance(theEntity); } @@ -251,8 +194,8 @@ private JobInstance createJobInstanceFromEntity(Batch2JobInstanceEntity theEntit private Batch2JobInstanceEntity createBatch2JobInstanceEntity() { Batch2JobInstanceEntity entity = new Batch2JobInstanceEntity(); entity.setId("id"); - entity.setStartTime(new Date(2000, 1, 2)); - entity.setEndTime(new Date(2000, 2, 3)); + entity.setStartTime(Date.from(LocalDate.of(2000, 1, 2).atStartOfDay().toInstant(ZoneOffset.UTC))); + entity.setEndTime(Date.from(LocalDate.of(2000, 2, 3).atStartOfDay().toInstant(ZoneOffset.UTC))); entity.setStatus(StatusEnum.COMPLETED); entity.setCancelled(true); entity.setFastTracking(true); diff --git a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/bulk/BulkGroupExportWithIndexedSearchParametersTest.java b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/bulk/BulkGroupExportWithIndexedSearchParametersTest.java index 3bae46528f58..f3f6047758a1 100644 --- a/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/bulk/BulkGroupExportWithIndexedSearchParametersTest.java +++ b/hapi-fhir-jpaserver-elastic-test-utilities/src/test/java/ca/uhn/fhir/jpa/bulk/BulkGroupExportWithIndexedSearchParametersTest.java @@ -17,8 +17,6 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Meta; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.RepeatedTest; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; diff --git a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java index 7e9343787d9c..72530cff00d9 100644 --- a/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java +++ b/hapi-fhir-jpaserver-model/src/main/java/ca/uhn/fhir/jpa/model/util/JpaConstants.java @@ -197,6 +197,11 @@ public class JpaConstants { * Parameter for the $export operation */ public static final String PARAM_EXPORT_TYPE_FILTER = "_typeFilter"; + + /** + * Parameter for the $export operation to identify binaries with a given identifier. + */ + public static final String PARAM_EXPORT_IDENTIFIER = "_exportId"; /** * Parameter for the $export operation */ @@ -206,6 +211,8 @@ public class JpaConstants { */ public static final String PARAM_EXPORT_PATIENT = "patient"; + + /** * Parameter for the $import operation */ @@ -289,6 +296,9 @@ public class JpaConstants { * IPS Generation operation URL */ public static final String SUMMARY_OPERATION_URL = "http://hl7.org/fhir/uv/ips/OperationDefinition/summary"; + public static final String BULK_META_EXTENSION_EXPORT_IDENTIFIER = "https://hapifhir.org/NamingSystem/bulk-export-identifier"; + public static final String BULK_META_EXTENSION_JOB_ID = "https://hapifhir.org/NamingSystem/bulk-export-job-id"; + public static final String BULK_META_EXTENSION_RESOURCE_TYPE = "https://hapifhir.org/NamingSystem/bulk-export-binary-resource-type"; /** * Non-instantiable diff --git a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/MockHapiTransactionService.java b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/MockHapiTransactionService.java index 22c7bdca9bce..fe17dee58793 100644 --- a/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/MockHapiTransactionService.java +++ b/hapi-fhir-jpaserver-test-dstu2/src/test/java/ca/uhn/fhir/jpa/search/MockHapiTransactionService.java @@ -1,22 +1,17 @@ package ca.uhn.fhir.jpa.search; import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; -import ca.uhn.fhir.rest.api.server.RequestDetails; -import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; -import org.springframework.transaction.annotation.Isolation; -import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.support.SimpleTransactionStatus; import org.springframework.transaction.support.TransactionCallback; -import javax.annotation.Nonnull; import javax.annotation.Nullable; -import java.util.concurrent.Callable; public class MockHapiTransactionService extends HapiTransactionService { @Nullable @Override protected T doExecute(ExecutionBuilder theExecutionBuilder, TransactionCallback theCallback) { - return theCallback.doInTransaction(null); + return theCallback.doInTransaction(new SimpleTransactionStatus()); } } diff --git a/hapi-fhir-jpaserver-test-r4/pom.xml b/hapi-fhir-jpaserver-test-r4/pom.xml index 6df28d58cea5..f7aacacd6bff 100644 --- a/hapi-fhir-jpaserver-test-r4/pom.xml +++ b/hapi-fhir-jpaserver-test-r4/pom.xml @@ -27,6 +27,12 @@ ${project.version} test + + ca.uhn.hapi.fhir + hapi-fhir-storage-batch2-test-utilities + ${project.version} + test + org.exparity hamcrest-date @@ -39,7 +45,7 @@ test - + diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java index f6739f3a1785..2d60ee307bd4 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2CoordinatorIT.java @@ -33,7 +33,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; @@ -56,6 +55,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -89,10 +89,14 @@ private static RunOutcome callLatch(PointcutLatch theLatch, StepExecutionDetails return RunOutcome.SUCCESS; } + @Override @BeforeEach - public void before() { + public void before() throws Exception { + super.before(); + myCompletionHandler = details -> {}; myWorkChannel = (LinkedBlockingChannel) myChannelFactory.getOrCreateReceiver(CHANNEL_NAME, JobWorkNotificationJsonMessage.class, new ChannelConsumerSettings()); + myDaoConfig.setJobFastTrackingEnabled(true); } @AfterEach @@ -106,13 +110,9 @@ public void fetchAllJobInstances_withValidInput_returnsPage() { // create a job // step 1 - IJobStepWorker first = (step, sink) -> { - return RunOutcome.SUCCESS; - }; + IJobStepWorker first = (step, sink) -> RunOutcome.SUCCESS; // final step - ILastJobStepWorker last = (step, sink) -> { - return RunOutcome.SUCCESS; - }; + ILastJobStepWorker last = (step, sink) -> RunOutcome.SUCCESS; // job definition String jobId = new Exception().getStackTrace()[0].getMethodName(); JobDefinition jd = JobDefinition.newBuilder() @@ -280,7 +280,7 @@ public void testJobDefinitionWithReductionStepIT(boolean theDelayReductionStepBo }; // step 3 - IReductionStepWorker last = new IReductionStepWorker() { + IReductionStepWorker last = new IReductionStepWorker<>() { private final ArrayList myOutput = new ArrayList<>(); @@ -288,6 +288,7 @@ public void testJobDefinitionWithReductionStepIT(boolean theDelayReductionStepBo private final AtomicInteger mySecondGate = new AtomicInteger(); + @Nonnull @Override public ChunkOutcome consume(ChunkExecutionDetails theChunkDetails) { myOutput.add(theChunkDetails.getData()); @@ -431,6 +432,7 @@ public void testUnknownException_KeepsInProgress_CanCancelManually() throws Inte JobInstanceStartRequest request = buildRequest(jobDefId); // execute + ourLog.info("Starting job"); myFirstStepLatch.setExpectedCount(1); Batch2JobStartResponse startResponse = myJobCoordinator.startInstance(request); String instanceId = startResponse.getJobId(); @@ -440,7 +442,9 @@ public void testUnknownException_KeepsInProgress_CanCancelManually() throws Inte myBatch2JobHelper.awaitJobInProgress(instanceId); // execute + ourLog.info("Cancel job {}", instanceId); myJobCoordinator.cancelInstance(instanceId); + ourLog.info("Cancel job {} done", instanceId); // validate myBatch2JobHelper.awaitJobCancelled(instanceId); @@ -487,13 +491,13 @@ public void testStepRunFailure_continuouslyThrows_marksJobFailed() { myFirstStepLatch.setExpectedCount(1); Batch2JobStartResponse response = myJobCoordinator.startInstance(request); JobInstance instance = myBatch2JobHelper.awaitJobHasStatus(response.getJobId(), - 12, // we want to wait a long time (2 min here) cause backoff is incremental + 30, // we want to wait a long time (2 min here) cause backoff is incremental StatusEnum.FAILED ); assertEquals(MAX_CHUNK_ERROR_COUNT + 1, counter.get()); - assertTrue(instance.getStatus() == StatusEnum.FAILED); + assertSame(StatusEnum.FAILED, instance.getStatus()); } @Nonnull diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java new file mode 100644 index 000000000000..1605ada7de8e --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobInstanceRepositoryTest.java @@ -0,0 +1,62 @@ +package ca.uhn.fhir.jpa.batch2; + +import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; +import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.Arrays; +import java.util.Date; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class Batch2JobInstanceRepositoryTest extends BaseJpaR4Test { + + @Autowired + IBatch2JobInstanceRepository myBatch2JobInstanceRepository; + + @ParameterizedTest + @CsvSource({ + "QUEUED, FAILED, QUEUED, true, normal transition", + "IN_PROGRESS, FAILED, QUEUED IN_PROGRESS ERRORED, true, normal transition with multiple prior", + "IN_PROGRESS, IN_PROGRESS, IN_PROGRESS, true, self transition to same state", + "QUEUED, QUEUED, QUEUED, true, normal transition", + "QUEUED, FAILED, IN_PROGRESS, false, blocked transition" + }) + void updateInstance_toState_fromState_whenAllowed(StatusEnum theCurrentState, StatusEnum theTargetState, String theAllowedPriorStatesString, boolean theExpectedSuccessFlag) { + Set theAllowedPriorStates = Arrays.stream(theAllowedPriorStatesString.trim().split(" +")).map(StatusEnum::valueOf).collect(Collectors.toSet()); + // given + Batch2JobInstanceEntity entity = new Batch2JobInstanceEntity(); + String jobId = UUID.randomUUID().toString(); + entity.setId(jobId); + entity.setStatus(theCurrentState); + entity.setCreateTime(new Date()); + entity.setDefinitionId("definition_id"); + myBatch2JobInstanceRepository.save(entity); + + // when + int changeCount = + runInTransaction(()-> + myBatch2JobInstanceRepository.updateInstanceStatusIfIn(jobId, theTargetState, theAllowedPriorStates)); + + // then + Batch2JobInstanceEntity readBack = runInTransaction(() -> + myBatch2JobInstanceRepository.findById(jobId).orElseThrow()); + if (theExpectedSuccessFlag) { + assertEquals(1, changeCount, "The change happened"); + assertEquals(theTargetState, readBack.getStatus()); + } else { + assertEquals(0, changeCount, "The change did not happened"); + assertEquals(theCurrentState, readBack.getStatus()); + } + + } + + +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java new file mode 100644 index 000000000000..b483c06fad44 --- /dev/null +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceDatabaseIT.java @@ -0,0 +1,552 @@ +package ca.uhn.fhir.jpa.batch2; + +import ca.uhn.fhir.batch2.api.IJobMaintenanceService; +import ca.uhn.fhir.batch2.api.IJobPersistence; +import ca.uhn.fhir.batch2.api.IJobStepWorker; +import ca.uhn.fhir.batch2.api.RunOutcome; +import ca.uhn.fhir.batch2.api.VoidModel; +import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; +import ca.uhn.fhir.batch2.maintenance.JobMaintenanceServiceImpl; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.fhir.batch2.model.JobWorkNotification; +import ca.uhn.fhir.batch2.model.JobWorkNotificationJsonMessage; +import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; +import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; +import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; +import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; +import ca.uhn.fhir.jpa.subscription.channel.api.ChannelConsumerSettings; +import ca.uhn.fhir.jpa.subscription.channel.api.ChannelProducerSettings; +import ca.uhn.fhir.jpa.subscription.channel.api.IChannelFactory; +import ca.uhn.fhir.jpa.subscription.channel.impl.LinkedBlockingChannel; +import ca.uhn.fhir.jpa.test.BaseJpaR4Test; +import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.util.JsonUtil; +import ca.uhn.test.concurrency.IPointcutLatch; +import ca.uhn.test.concurrency.PointcutLatch; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Optional; + +import static ca.uhn.fhir.batch2.config.BaseBatch2Config.CHANNEL_NAME; +import static org.apache.commons.lang3.StringUtils.isBlank; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasSize; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class Batch2JobMaintenanceDatabaseIT extends BaseJpaR4Test { + private static final Logger ourLog = LoggerFactory.getLogger(Batch2JobMaintenanceDatabaseIT.class); + + public static final int TEST_JOB_VERSION = 1; + public static final String FIRST = "FIRST"; + public static final String SECOND = "SECOND"; + public static final String LAST = "LAST"; + private static final String JOB_DEF_ID = "test-job-definition"; + private static final JobDefinition ourJobDef = buildJobDefinition(); + private static final String TEST_INSTANCE_ID = "test-instance-id"; + + @Autowired + JobDefinitionRegistry myJobDefinitionRegistry; + @Autowired + IJobMaintenanceService myJobMaintenanceService; + @Autowired + private IChannelFactory myChannelFactory; + + @Autowired + IJobPersistence myJobPersistence; + @Autowired + IBatch2JobInstanceRepository myJobInstanceRepository; + @Autowired + IBatch2WorkChunkRepository myWorkChunkRepository; + + private LinkedBlockingChannel myWorkChannel; + private final List myStackTraceElements = new ArrayList<>(); + private TransactionTemplate myTxTemplate; + private final MyChannelInterceptor myChannelInterceptor = new MyChannelInterceptor(); + + @BeforeEach + public void before() { + myWorkChunkRepository.deleteAll(); + myJobInstanceRepository.deleteAll(); + + myJobDefinitionRegistry.addJobDefinition(ourJobDef); + myWorkChannel = (LinkedBlockingChannel) myChannelFactory.getOrCreateProducer(CHANNEL_NAME, JobWorkNotificationJsonMessage.class, new ChannelProducerSettings()); + JobMaintenanceServiceImpl jobMaintenanceService = (JobMaintenanceServiceImpl) myJobMaintenanceService; + jobMaintenanceService.setMaintenanceJobStartedCallback(() -> { + ourLog.info("Batch maintenance job started"); + myStackTraceElements.add(Thread.currentThread().getStackTrace()); + }); + + myTxTemplate = new TransactionTemplate(myTxManager); + storeNewInstance(ourJobDef); + + myWorkChannel = (LinkedBlockingChannel) myChannelFactory.getOrCreateReceiver(CHANNEL_NAME, JobWorkNotificationJsonMessage.class, new ChannelConsumerSettings()); + myChannelInterceptor.clear(); + myWorkChannel.addInterceptor(myChannelInterceptor); + } + + @AfterEach + public void after() { + ourLog.debug("Maintenance traces: {}", myStackTraceElements); + myWorkChannel.clearInterceptorsForUnitTest(); + JobMaintenanceServiceImpl jobMaintenanceService = (JobMaintenanceServiceImpl) myJobMaintenanceService; + jobMaintenanceService.setMaintenanceJobStartedCallback(() -> { + }); + } + + @Test + public void runMaintenancePass_noChunks_noChange() { + assertInstanceCount(1); + myJobMaintenanceService.runMaintenancePass(); + assertInstanceCount(1); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + } + + + @Test + public void runMaintenancePass_SingleQueuedChunk_noChange() { + WorkChunkExpectation expectation = new WorkChunkExpectation( + "chunk1, FIRST, QUEUED", + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + } + + @Test + public void runMaintenancePass_SingleInProgressChunk_noChange() { + WorkChunkExpectation expectation = new WorkChunkExpectation( + "chunk1, FIRST, IN_PROGRESS", + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + } + + @Test + public void runMaintenancePass_SingleCompleteChunk_notifiesAndChangesGatedStep() throws InterruptedException { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + chunk2, SECOND, QUEUED + """, + """ + chunk2 + """ + ); + + expectation.storeChunks(); + myChannelInterceptor.setExpectedCount(1); + myJobMaintenanceService.runMaintenancePass(); + myChannelInterceptor.awaitExpected(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + assertCurrentGatedStep(SECOND); + } + + @Test + public void runMaintenancePass_DoubleCompleteChunk_notifiesAndChangesGatedStep() throws InterruptedException { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + chunk2, FIRST, COMPLETED + chunk3, SECOND, QUEUED + chunk4, SECOND, QUEUED + """, """ + chunk3 + chunk4 + """ + ); + + expectation.storeChunks(); + myChannelInterceptor.setExpectedCount(2); + myJobMaintenanceService.runMaintenancePass(); + myChannelInterceptor.awaitExpected(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + assertCurrentGatedStep(SECOND); + } + + @Test + public void runMaintenancePass_DoubleIncompleteChunk_noChange() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + chunk2, FIRST, IN_PROGRESS + chunk3, SECOND, QUEUED + chunk4, SECOND, QUEUED + """, + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.IN_PROGRESS); + assertCurrentGatedStep(FIRST); + } + + @Test + public void runMaintenancePass_allStepsComplete_jobCompletes() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + chunk3, SECOND, COMPLETED + chunk4, SECOND, COMPLETED + chunk5, SECOND, COMPLETED + chunk6, SECOND, COMPLETED + chunk7, LAST, COMPLETED + """, + "" + ); + + expectation.storeChunks(); + + + myJobMaintenanceService.runMaintenancePass(); + + + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.COMPLETED); + + } + + + /** + * If the first step doesn't produce any work chunks, then + * the instance should be marked as complete right away. + */ + @Test + public void testPerformStep_FirstStep_NoWorkChunksProduced() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + """, + "" + ); + + expectation.storeChunks(); + + myJobMaintenanceService.runMaintenancePass(); + myJobMaintenanceService.runMaintenancePass(); + myJobMaintenanceService.runMaintenancePass(); + myJobMaintenanceService.runMaintenancePass(); + + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.COMPLETED); + } + + + /** + * Once all chunks are complete, the job should complete even if a step has no work. + * the instance should be marked as complete right away. + */ + @Test + public void testPerformStep_secondStep_NoWorkChunksProduced() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, COMPLETED + chunk3, SECOND, COMPLETED + chunk4, SECOND, COMPLETED + """, + "" + ); + + expectation.storeChunks(); + + myJobMaintenanceService.runMaintenancePass(); + myJobMaintenanceService.runMaintenancePass(); + myJobMaintenanceService.runMaintenancePass(); + + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.COMPLETED); + } + + // TODO MB Ken and Nathan created these. Do we want to make them real? + @Test + @Disabled("future plans") + public void runMaintenancePass_MultipleStepsInProgress_CancelsInstance() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, FIRST, IN_PROGRESS + chunk2, SECOND, IN_PROGRESS + """, + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.FAILED); + assertError("IN_PROGRESS Chunks found in both the FIRST and SECOND step."); + } + + @Test + @Disabled("future plans") + public void runMaintenancePass_MultipleOtherStepsInProgress_CancelsInstance() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ + chunk1, SECOND, IN_PROGRESS + chunk2, LAST, IN_PROGRESS + """, + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.FAILED); + assertError("IN_PROGRESS Chunks found both the SECOND and LAST step."); + } + + @Test + @Disabled("future plans") + public void runMaintenancePass_MultipleStepsQueued_CancelsInstance() { + assertCurrentGatedStep(FIRST); + + WorkChunkExpectation expectation = new WorkChunkExpectation( + """ +chunk1, FIRST, COMPLETED +chunk2, SECOND, QUEUED +chunk3, LAST, QUEUED +""", + "" + ); + + expectation.storeChunks(); + myJobMaintenanceService.runMaintenancePass(); + expectation.assertNotifications(); + + assertInstanceStatus(StatusEnum.FAILED); + assertError("QUEUED Chunks found in both the SECOND and LAST step."); + } + + private void assertError(String theExpectedErrorMessage) { + Optional instance = myJobInstanceRepository.findById(TEST_INSTANCE_ID); + assertTrue(instance.isPresent()); + assertEquals(theExpectedErrorMessage, instance.get().getErrorMessage()); + } + + + private void assertCurrentGatedStep(String theNextStepId) { + Optional instance = myJobPersistence.fetchInstance(TEST_INSTANCE_ID); + assertTrue(instance.isPresent()); + assertEquals(theNextStepId, instance.get().getCurrentGatedStepId()); + } + + @Nonnull + private static Batch2WorkChunkEntity buildWorkChunkEntity(String theChunkId, String theStepId, WorkChunkStatusEnum theStatus) { + Batch2WorkChunkEntity workChunk = new Batch2WorkChunkEntity(); + workChunk.setId(theChunkId); + workChunk.setJobDefinitionId(JOB_DEF_ID); + workChunk.setStatus(theStatus); + workChunk.setJobDefinitionVersion(TEST_JOB_VERSION); + workChunk.setCreateTime(new Date()); + workChunk.setInstanceId(TEST_INSTANCE_ID); + workChunk.setTargetStepId(theStepId); + if (!theStatus.isIncomplete()) { + workChunk.setEndTime(new Date()); + } + + return workChunk; + } + + + @Nonnull + private static JobDefinition buildJobDefinition() { + IJobStepWorker firstStep = (step, sink) -> { + ourLog.info("First step for chunk {}", step.getChunkId()); + return RunOutcome.SUCCESS; + }; + IJobStepWorker secondStep = (step, sink) -> { + ourLog.info("Second step for chunk {}", step.getChunkId()); + return RunOutcome.SUCCESS; + }; + IJobStepWorker lastStep = (step, sink) -> { + ourLog.info("Last step for chunk {}", step.getChunkId()); + return RunOutcome.SUCCESS; + }; + + JobDefinition definition = buildGatedJobDefinition(firstStep, secondStep, lastStep); + return definition; + } + + private void storeNewInstance(JobDefinition theJobDefinition) { + Batch2JobInstanceEntity entity = new Batch2JobInstanceEntity(); + entity.setId(TEST_INSTANCE_ID); + entity.setStatus(StatusEnum.IN_PROGRESS); + entity.setDefinitionId(theJobDefinition.getJobDefinitionId()); + entity.setDefinitionVersion(theJobDefinition.getJobDefinitionVersion()); + entity.setParams(JsonUtil.serializeOrInvalidRequest(new TestJobParameters())); + entity.setCurrentGatedStepId(FIRST); + entity.setCreateTime(new Date()); + + myTxTemplate.executeWithoutResult(t -> myJobInstanceRepository.save(entity)); + } + + private void assertInstanceCount(int size) { + assertThat(myJobPersistence.fetchInstancesByJobDefinitionId(JOB_DEF_ID, 100, 0), hasSize(size)); + } + + + private void assertInstanceStatus(StatusEnum theInProgress) { + Optional instance = myJobInstanceRepository.findById(TEST_INSTANCE_ID); + assertTrue(instance.isPresent()); + assertEquals(theInProgress, instance.get().getStatus()); + } + @Nonnull + private static JobDefinition buildGatedJobDefinition(IJobStepWorker theFirstStep, IJobStepWorker theSecondStep, IJobStepWorker theLastStep) { + return JobDefinition.newBuilder() + .setJobDefinitionId(JOB_DEF_ID) + .setJobDescription("test job") + .setJobDefinitionVersion(TEST_JOB_VERSION) + .setParametersType(TestJobParameters.class) + .gatedExecution() + .addFirstStep( + FIRST, + "Test first step", + FirstStepOutput.class, + theFirstStep + ) + .addIntermediateStep( + SECOND, + "Test second step", + SecondStepOutput.class, + theSecondStep + ) + .addLastStep( + LAST, + "Test last step", + theLastStep + ) + .completionHandler(details -> { + }) + .build(); + } + + static class TestJobParameters implements IModelJson { + TestJobParameters() { + } + } + + static class FirstStepOutput implements IModelJson { + FirstStepOutput() { + } + } + + static class SecondStepOutput implements IModelJson { + SecondStepOutput() { + } + } + + private class WorkChunkExpectation { + private final List myInputChunks = new ArrayList<>(); + private final List myExpectedChunkIdNotifications = new ArrayList<>(); + public WorkChunkExpectation(String theInput, String theOutputChunkIds) { + String[] inputLines = theInput.split("\n"); + for (String next : inputLines) { + String[] parts = next.split(","); + Batch2WorkChunkEntity e = buildWorkChunkEntity(parts[0].trim(), parts[1].trim(), WorkChunkStatusEnum.valueOf(parts[2].trim())); + myInputChunks.add(e); + } + if (!isBlank(theOutputChunkIds)) { + String[] outputLines = theOutputChunkIds.split("\n"); + for (String next : outputLines) { + myExpectedChunkIdNotifications.add(next.trim()); + } + } + } + + public void storeChunks() { + myTxTemplate.executeWithoutResult(t -> myWorkChunkRepository.saveAll(myInputChunks)); + } + + public void assertNotifications() { + assertThat(myChannelInterceptor.getReceivedChunkIds(), containsInAnyOrder(myExpectedChunkIdNotifications.toArray())); + } + } + + private static class MyChannelInterceptor implements ChannelInterceptor, IPointcutLatch { + PointcutLatch myPointcutLatch = new PointcutLatch("BATCH CHUNK MESSAGE RECEIVED"); + List myReceivedChunkIds = new ArrayList<>(); + @Override + public Message preSend(@Nonnull Message message, @Nonnull MessageChannel channel) { + ourLog.info("Sending message: {}", message); + JobWorkNotification notification = ((JobWorkNotificationJsonMessage) message).getPayload(); + myReceivedChunkIds.add(notification.getChunkId()); + myPointcutLatch.call(message); + return message; + } + + @Override + public void clear() { + myPointcutLatch.clear(); + } + + @Override + public void setExpectedCount(int count) { + myPointcutLatch.setExpectedCount(count); + } + + @Override + public List awaitExpected() throws InterruptedException { + return myPointcutLatch.awaitExpected(); + } + + List getReceivedChunkIds() { + return myReceivedChunkIds; + } + } +} diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java index 6d549b18cd2c..d81a49c3c2c2 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/Batch2JobMaintenanceIT.java @@ -39,6 +39,17 @@ import static ca.uhn.fhir.batch2.config.BaseBatch2Config.CHANNEL_NAME; import static org.junit.jupiter.api.Assertions.assertTrue; +/** + * The on-enter actions are defined in + * {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater#handleStatusChange} + * {@link ca.uhn.fhir.batch2.progress.InstanceProgress#updateStatus(JobInstance)} + * {@link JobInstanceProcessor#cleanupInstance()} + + * For chunks: + * {@link ca.uhn.fhir.jpa.batch2.JpaJobPersistenceImpl#onWorkChunkCreate} + * {@link JpaJobPersistenceImpl#onWorkChunkDequeue(String)} + * Chunk execution {@link ca.uhn.fhir.batch2.coordinator.StepExecutor#executeStep} +*/ @TestPropertySource(properties = { UnregisterScheduledProcessor.SCHEDULING_DISABLED_EQUALS_FALSE }) diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataErrorAbuseTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java similarity index 86% rename from hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataErrorAbuseTest.java rename to hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java index 1a5808bfe0e3..d4a64d910c43 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkDataErrorAbuseTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/BulkDataErrorAbuseTest.java @@ -1,10 +1,12 @@ -package ca.uhn.fhir.jpa.bulk; +package ca.uhn.fhir.jpa.batch2; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.BulkExportJobResults; +import ca.uhn.fhir.jpa.api.model.BulkExportParameters; import ca.uhn.fhir.jpa.api.svc.IBatch2JobRunner; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; +import ca.uhn.fhir.jpa.test.config.TestR4Config; import ca.uhn.fhir.jpa.util.BulkExportUtils; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; @@ -17,6 +19,7 @@ import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.Patient; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.slf4j.Logger; @@ -60,9 +63,15 @@ public class BulkDataErrorAbuseTest extends BaseResourceProviderR4Test { @Autowired private IBatch2JobRunner myJobRunner; + @BeforeEach + void beforeEach() { + afterPurgeDatabase(); + } + @AfterEach void afterEach() { myDaoConfig.setIndexMissingFields(DaoConfig.IndexEnabledEnum.DISABLED); + ourLog.info("BulkDataErrorAbuseTest.afterEach()"); } @Test @@ -71,16 +80,16 @@ public void testGroupBulkExportNotInGroup_DoesNotShowUp() throws InterruptedExce } /** - * This test is disabled because it never actually exists. Run it if you want to ensure + * This test is disabled because it never actually exits. Run it if you want to ensure * that changes to the Bulk Export Batch2 task haven't affected our ability to successfully - * run endless parallel jobs. If you run it for a few minutes and it never stops on its own, + * run endless parallel jobs. If you run it for a few minutes, and it never stops on its own, * you are good. *

* The enabled test above called {@link #testGroupBulkExportNotInGroup_DoesNotShowUp()} does * run with the build and runs 100 jobs. */ @Test - @Disabled + @Disabled("for manual debugging") public void testNonStopAbuseBatch2BulkExportStressTest() throws InterruptedException, ExecutionException { duAbuseTest(Integer.MAX_VALUE); } @@ -121,7 +130,8 @@ private void duAbuseTest(int taskExecutions) throws InterruptedException, Execut options.setOutputFormat(Constants.CT_FHIR_NDJSON); BlockingQueue workQueue = new LinkedBlockingQueue<>(); - ExecutorService executorService = new ThreadPoolExecutor(10, 10, + int workerCount = TestR4Config.ourMaxThreads - 1; // apply a little connection hunger, but not starvation. + ExecutorService executorService = new ThreadPoolExecutor(workerCount, workerCount, 0L, TimeUnit.MILLISECONDS, workQueue); @@ -147,8 +157,8 @@ private void duAbuseTest(int taskExecutions) throws InterruptedException, Execut })); // Don't let the list of futures grow so big we run out of memory - if (futures.size() > 200) { - while (futures.size() > 100) { + if (futures.size() > 1000) { + while (futures.size() > 500) { // This should always return true, but it'll throw an exception if we failed assertTrue(futures.remove(0).get()); } @@ -157,6 +167,12 @@ private void duAbuseTest(int taskExecutions) throws InterruptedException, Execut ourLog.info("Done creating tasks, waiting for task completion"); + // wait for completion to avoid stranding background tasks. + executorService.shutdown(); + assertTrue(executorService.awaitTermination(60, TimeUnit.SECONDS), "Finished before timeout"); + + // verify that all requests succeeded + ourLog.info("All tasks complete. Verify results."); for (var next : futures) { // This should always return true, but it'll throw an exception if we failed assertTrue(next.get()); @@ -217,7 +233,9 @@ private void verifyBulkExportResults(String theInstanceId, List theConta } private String startJob(BulkDataExportOptions theOptions) { - Batch2JobStartResponse startResponse = myJobRunner.startNewJob(BulkExportUtils.createBulkExportJobParametersFromExportOptions(theOptions)); + BulkExportParameters startRequest = BulkExportUtils.createBulkExportJobParametersFromExportOptions(theOptions); + startRequest.setUseExistingJobsFirst(false); + Batch2JobStartResponse startResponse = myJobRunner.startNewJob(startRequest); assertNotNull(startResponse); return startResponse.getJobId(); } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java index 9302986767a2..a91a03c9cad9 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/batch2/JpaJobPersistenceImplTest.java @@ -1,21 +1,24 @@ package ca.uhn.fhir.jpa.batch2; - import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobOperationResultJson; -import ca.uhn.fhir.batch2.coordinator.BatchWorkChunk; import ca.uhn.fhir.batch2.jobs.imprt.NdJsonFileJson; import ca.uhn.fhir.batch2.model.JobInstance; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository; import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; import ca.uhn.fhir.jpa.test.BaseJpaR4Test; import ca.uhn.fhir.util.JsonUtil; +import ca.uhn.hapi.fhir.batch2.test.AbstractIJobPersistenceSpecificationTest; import com.google.common.collect.Iterators; import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestMethodOrder; import org.junit.jupiter.params.ParameterizedTest; @@ -23,10 +26,13 @@ import org.junit.jupiter.params.provider.MethodSource; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.domain.PageRequest; +import org.springframework.transaction.PlatformTransactionManager; import javax.annotation.Nonnull; +import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Date; import java.util.HashSet; @@ -57,7 +63,6 @@ public class JpaJobPersistenceImplTest extends BaseJpaR4Test { public static final int JOB_DEF_VER = 1; public static final int SEQUENCE_NUMBER = 1; public static final String CHUNK_DATA = "{\"key\":\"value\"}"; - public static final String INSTANCE_ID = "instance-id"; @Autowired private IJobPersistence mySvc; @@ -89,8 +94,8 @@ public void testDeleteInstance() { } private String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData) { - BatchWorkChunk batchWorkChunk = new BatchWorkChunk(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData); - return mySvc.storeWorkChunk(batchWorkChunk); + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData); + return mySvc.onWorkChunkCreate(batchWorkChunk); } @Test @@ -145,8 +150,8 @@ public void testFetchInstanceWithStatusAndCutoff_statues() { final String completedId = storeJobInstanceAndUpdateWithEndTime(StatusEnum.COMPLETED, 1); final String failedId = storeJobInstanceAndUpdateWithEndTime(StatusEnum.FAILED, 1); - final String erroredId = storeJobInstanceAndUpdateWithEndTime(StatusEnum.ERRORED, 1); final String cancelledId = storeJobInstanceAndUpdateWithEndTime(StatusEnum.CANCELLED, 1); + storeJobInstanceAndUpdateWithEndTime(StatusEnum.ERRORED, 1); storeJobInstanceAndUpdateWithEndTime(StatusEnum.QUEUED, 1); storeJobInstanceAndUpdateWithEndTime(StatusEnum.IN_PROGRESS, 1); storeJobInstanceAndUpdateWithEndTime(StatusEnum.FINALIZE, 1); @@ -160,7 +165,7 @@ public void testFetchInstanceWithStatusAndCutoff_statues() { final List jobInstancesByCutoff = mySvc.fetchInstances(JOB_DEFINITION_ID, StatusEnum.getEndedStatuses(), cutoffDate, PageRequest.of(0, 100)); - assertEquals(Set.of(completedId, failedId, erroredId, cancelledId), + assertEquals(Set.of(completedId, failedId, cancelledId), jobInstancesByCutoff.stream() .map(JobInstance::getInstanceId) .collect(Collectors.toUnmodifiableSet())); @@ -221,20 +226,20 @@ public void testFetchInstanceWithStatusAndCutoff_pages() { @ParameterizedTest @MethodSource("provideStatuses") - public void testStartChunkOnlyWorksOnValidChunks(StatusEnum theStatus, boolean theShouldBeStartedByConsumer) { + public void testStartChunkOnlyWorksOnValidChunks(WorkChunkStatusEnum theStatus, boolean theShouldBeStartedByConsumer) { // Setup JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); - BatchWorkChunk batchWorkChunk = new BatchWorkChunk(JOB_DEFINITION_ID, JOB_DEF_VER, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); - String chunkId = mySvc.storeWorkChunk(batchWorkChunk); + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(JOB_DEFINITION_ID, JOB_DEF_VER, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); + String chunkId = mySvc.onWorkChunkCreate(batchWorkChunk); Optional byId = myWorkChunkRepository.findById(chunkId); Batch2WorkChunkEntity entity = byId.get(); entity.setStatus(theStatus); myWorkChunkRepository.save(entity); // Execute - Optional workChunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(chunkId); + Optional workChunk = mySvc.onWorkChunkDequeue(chunkId); // Verify boolean chunkStarted = workChunk.isPresent(); @@ -246,14 +251,8 @@ public void testCancelInstance() { JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); - runInTransaction(() -> { - Batch2JobInstanceEntity instanceEntity = myJobInstanceRepository.findById(instanceId).orElseThrow(IllegalStateException::new); - assertEquals(StatusEnum.QUEUED, instanceEntity.getStatus()); - instanceEntity.setCancelled(true); - myJobInstanceRepository.save(instanceEntity); - }); - JobOperationResultJson result = mySvc.cancelInstance(instanceId); + assertTrue(result.getSuccess()); assertEquals("Job instance <" + instanceId + "> successfully cancelled.", result.getMessage()); @@ -290,6 +289,24 @@ void testFetchInstancesByJobDefinitionIdAndStatus() { assertEquals(instanceId, foundInstances.get(0).getInstanceId()); } + /** + * Test bodies are defined in {@link AbstractIJobPersistenceSpecificationTest}. + * The nested test suite runs those tests here in a JPA context. + */ + @Nested + class Batch2SpecTest extends AbstractIJobPersistenceSpecificationTest { + + @Override + protected PlatformTransactionManager getTxManager() { + return JpaJobPersistenceImplTest.this.getTxManager(); + } + + @Override + protected WorkChunk freshFetchWorkChunk(String chunkId) { + return JpaJobPersistenceImplTest.this.freshFetchWorkChunk(chunkId); + } + } + @Test public void testFetchChunks() { JobInstance instance = createInstance(); @@ -355,13 +372,12 @@ public void testStoreAndFetchWorkChunk_NoData() { String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, null); - WorkChunk chunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(id).orElseThrow(IllegalArgumentException::new); + WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); assertNull(chunk.getData()); } @Test void testStoreAndFetchChunksForInstance_NoData() { - //wipmb here // given JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); @@ -370,14 +386,12 @@ void testStoreAndFetchChunksForInstance_NoData() { String erroredId = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 1, "some more data"); String completedId = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 2, "some more data"); - mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(erroredId); - MarkWorkChunkAsErrorRequest parameters = new MarkWorkChunkAsErrorRequest(); - parameters.setChunkId(erroredId); - parameters.setErrorMsg("Our error message"); - mySvc.markWorkChunkAsErroredAndIncrementErrorCount(parameters); + mySvc.onWorkChunkDequeue(erroredId); + WorkChunkErrorEvent parameters = new WorkChunkErrorEvent(erroredId, "Our error message"); + mySvc.onWorkChunkError(parameters); - mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(completedId); - mySvc.markWorkChunkAsCompletedAndClearData(instanceId, completedId, 11); + mySvc.onWorkChunkDequeue(completedId); + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(completedId, 11, 0)); // when Iterator workChunks = mySvc.fetchAllWorkChunksIterator(instanceId, false); @@ -396,7 +410,7 @@ void testStoreAndFetchChunksForInstance_NoData() { assertEquals(instanceId, workChunk.getInstanceId()); assertEquals(TARGET_STEP_ID, workChunk.getTargetStepId()); assertEquals(0, workChunk.getSequence()); - assertEquals(StatusEnum.QUEUED, workChunk.getStatus()); + assertEquals(WorkChunkStatusEnum.QUEUED, workChunk.getStatus()); assertNotNull(workChunk.getCreateTime()); @@ -410,7 +424,7 @@ void testStoreAndFetchChunksForInstance_NoData() { { WorkChunk workChunk1 = chunks.get(1); - assertEquals(StatusEnum.ERRORED, workChunk1.getStatus()); + assertEquals(WorkChunkStatusEnum.ERRORED, workChunk1.getStatus()); assertEquals("Our error message", workChunk1.getErrorMessage()); assertEquals(1, workChunk1.getErrorCount()); assertEquals(null, workChunk1.getRecordsProcessed()); @@ -419,7 +433,7 @@ void testStoreAndFetchChunksForInstance_NoData() { { WorkChunk workChunk2 = chunks.get(2); - assertEquals(StatusEnum.COMPLETED, workChunk2.getStatus()); + assertEquals(WorkChunkStatusEnum.COMPLETED, workChunk2.getStatus()); assertNotNull(workChunk2.getEndTime()); assertEquals(11, workChunk2.getRecordsProcessed()); assertNull(workChunk2.getErrorMessage()); @@ -436,16 +450,16 @@ public void testStoreAndFetchWorkChunk_WithData() { String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); assertNotNull(id); - runInTransaction(() -> assertEquals(StatusEnum.QUEUED, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); - WorkChunk chunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(id).orElseThrow(IllegalArgumentException::new); + WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); assertEquals(36, chunk.getInstanceId().length()); assertEquals(JOB_DEFINITION_ID, chunk.getJobDefinitionId()); assertEquals(JOB_DEF_VER, chunk.getJobDefinitionVersion()); - assertEquals(StatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); assertEquals(CHUNK_DATA, chunk.getData()); - runInTransaction(() -> assertEquals(StatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(id).orElseThrow(IllegalArgumentException::new).getStatus())); } @Test @@ -455,26 +469,26 @@ public void testMarkChunkAsCompleted_Success() { String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, CHUNK_DATA); assertNotNull(chunkId); - runInTransaction(() -> assertEquals(StatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); sleepUntilTimeChanges(); - WorkChunk chunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(chunkId).orElseThrow(IllegalArgumentException::new); + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); - assertEquals(StatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); assertNotNull(chunk.getCreateTime()); assertNotNull(chunk.getStartTime()); assertNull(chunk.getEndTime()); assertNull(chunk.getRecordsProcessed()); assertNotNull(chunk.getData()); - runInTransaction(() -> assertEquals(StatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); sleepUntilTimeChanges(); - mySvc.markWorkChunkAsCompletedAndClearData(INSTANCE_ID, chunkId, 50); + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 50, 0)); runInTransaction(() -> { Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); - assertEquals(StatusEnum.COMPLETED, entity.getStatus()); + assertEquals(WorkChunkStatusEnum.COMPLETED, entity.getStatus()); assertEquals(50, entity.getRecordsProcessed()); assertNotNull(entity.getCreateTime()); assertNotNull(entity.getStartTime()); @@ -485,34 +499,13 @@ public void testMarkChunkAsCompleted_Success() { }); } - @Test - public void testIncrementWorkChunkErrorCount() { - // Setup - - JobInstance instance = createInstance(); - String instanceId = mySvc.storeNewInstance(instance); - String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - assertNotNull(chunkId); - - // Execute - - mySvc.incrementWorkChunkErrorCount(chunkId, 2); - mySvc.incrementWorkChunkErrorCount(chunkId, 3); - - // Verify - - List chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 100, 0); - assertEquals(1, chunks.size()); - assertEquals(5, chunks.get(0).getErrorCount()); - } - @Test public void testGatedAdvancementByStatus() { // Setup JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - mySvc.markWorkChunkAsCompletedAndClearData(INSTANCE_ID, chunkId, 0); + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 0, 0)); boolean canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); assertTrue(canAdvance); @@ -524,18 +517,18 @@ public void testGatedAdvancementByStatus() { assertFalse(canAdvance); //Toggle it to complete - mySvc.markWorkChunkAsCompletedAndClearData(INSTANCE_ID, newChunkId, 0); + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(newChunkId, 50, 0)); canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); assertTrue(canAdvance); //Create a new chunk and set it in progress. String newerChunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); - mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(newerChunkId); + mySvc.onWorkChunkDequeue(newerChunkId); canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); assertFalse(canAdvance); //Toggle IN_PROGRESS to complete - mySvc.markWorkChunkAsCompletedAndClearData(INSTANCE_ID, newerChunkId, 0); + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(newerChunkId, 50, 0)); canAdvance = mySvc.canAdvanceInstanceToNextStep(instanceId, STEP_CHUNK_ID); assertTrue(canAdvance); } @@ -547,21 +540,21 @@ public void testMarkChunkAsCompleted_Error() { String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); assertNotNull(chunkId); - runInTransaction(() -> assertEquals(StatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); sleepUntilTimeChanges(); - WorkChunk chunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(chunkId).orElseThrow(IllegalArgumentException::new); + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); - assertEquals(StatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); sleepUntilTimeChanges(); - MarkWorkChunkAsErrorRequest request = new MarkWorkChunkAsErrorRequest().setChunkId(chunkId).setErrorMsg("This is an error message"); - mySvc.markWorkChunkAsErroredAndIncrementErrorCount(request); + WorkChunkErrorEvent request = new WorkChunkErrorEvent(chunkId).setErrorMsg("This is an error message"); + mySvc.onWorkChunkError(request); runInTransaction(() -> { Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); - assertEquals(StatusEnum.ERRORED, entity.getStatus()); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); assertEquals("This is an error message", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); assertNotNull(entity.getStartTime()); @@ -573,11 +566,11 @@ public void testMarkChunkAsCompleted_Error() { // Mark errored again - MarkWorkChunkAsErrorRequest request2 = new MarkWorkChunkAsErrorRequest().setChunkId(chunkId).setErrorMsg("This is an error message 2"); - mySvc.markWorkChunkAsErroredAndIncrementErrorCount(request2); + WorkChunkErrorEvent request2 = new WorkChunkErrorEvent(chunkId).setErrorMsg("This is an error message 2"); + mySvc.onWorkChunkError(request2); runInTransaction(() -> { Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); - assertEquals(StatusEnum.ERRORED, entity.getStatus()); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); assertEquals("This is an error message 2", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); assertNotNull(entity.getStartTime()); @@ -599,20 +592,20 @@ public void testMarkChunkAsCompleted_Fail() { String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); assertNotNull(chunkId); - runInTransaction(() -> assertEquals(StatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new).getStatus())); sleepUntilTimeChanges(); - WorkChunk chunk = mySvc.fetchWorkChunkSetStartTimeAndMarkInProgress(chunkId).orElseThrow(IllegalArgumentException::new); + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); - assertEquals(StatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); sleepUntilTimeChanges(); - mySvc.markWorkChunkAsFailed(chunkId, "This is an error message"); + mySvc.onWorkChunkFailed(chunkId, "This is an error message"); runInTransaction(() -> { Batch2WorkChunkEntity entity = myWorkChunkRepository.findById(chunkId).orElseThrow(IllegalArgumentException::new); - assertEquals(StatusEnum.FAILED, entity.getStatus()); + assertEquals(WorkChunkStatusEnum.FAILED, entity.getStatus()); assertEquals("This is an error message", entity.getErrorMessage()); assertNotNull(entity.getCreateTime()); assertNotNull(entity.getStartTime()); @@ -635,48 +628,13 @@ public void testMarkInstanceAsCompleted() { }); } - @Test - public void testUpdateInstance() { - String instanceId = mySvc.storeNewInstance(createInstance()); - - JobInstance instance = mySvc.fetchInstance(instanceId).orElseThrow(IllegalArgumentException::new); - assertEquals(instanceId, instance.getInstanceId()); - assertFalse(instance.isWorkChunksPurged()); - - instance.setStartTime(new Date()); - sleepUntilTimeChanges(); - instance.setEndTime(new Date()); - instance.setCombinedRecordsProcessed(100); - instance.setCombinedRecordsProcessedPerSecond(22.0); - instance.setWorkChunksPurged(true); - instance.setProgress(0.5d); - instance.setErrorCount(3); - instance.setEstimatedTimeRemaining("32d"); - - mySvc.updateInstance(instance); - - runInTransaction(() -> { - Batch2JobInstanceEntity entity = myJobInstanceRepository.findById(instanceId).orElseThrow(IllegalArgumentException::new); - assertEquals(instance.getStartTime().getTime(), entity.getStartTime().getTime()); - assertEquals(instance.getEndTime().getTime(), entity.getEndTime().getTime()); - }); - - JobInstance finalInstance = mySvc.fetchInstance(instanceId).orElseThrow(IllegalArgumentException::new); - assertEquals(instanceId, finalInstance.getInstanceId()); - assertEquals(0.5d, finalInstance.getProgress()); - assertTrue(finalInstance.isWorkChunksPurged()); - assertEquals(3, finalInstance.getErrorCount()); - assertEquals(instance.getReport(), finalInstance.getReport()); - assertEquals(instance.getEstimatedTimeRemaining(), finalInstance.getEstimatedTimeRemaining()); - } - @Test public void markWorkChunksWithStatusAndWipeData_marksMultipleChunksWithStatus_asExpected() { JobInstance instance = createInstance(); String instanceId = mySvc.storeNewInstance(instance); ArrayList chunkIds = new ArrayList<>(); for (int i = 0; i < 10; i++) { - BatchWorkChunk chunk = new BatchWorkChunk( + WorkChunkCreateEvent chunk = new WorkChunkCreateEvent( "defId", 1, "stepId", @@ -684,21 +642,29 @@ public void markWorkChunksWithStatusAndWipeData_marksMultipleChunksWithStatus_as 0, "{}" ); - String id = mySvc.storeWorkChunk(chunk); + String id = mySvc.onWorkChunkCreate(chunk); chunkIds.add(id); } - runInTransaction(() -> mySvc.markWorkChunksWithStatusAndWipeData(instance.getInstanceId(), chunkIds, StatusEnum.COMPLETED, null)); + runInTransaction(() -> mySvc.markWorkChunksWithStatusAndWipeData(instance.getInstanceId(), chunkIds, WorkChunkStatusEnum.COMPLETED, null)); Iterator reducedChunks = mySvc.fetchAllWorkChunksIterator(instanceId, true); while (reducedChunks.hasNext()) { WorkChunk reducedChunk = reducedChunks.next(); assertTrue(chunkIds.contains(reducedChunk.getId())); - assertEquals(StatusEnum.COMPLETED, reducedChunk.getStatus()); + assertEquals(WorkChunkStatusEnum.COMPLETED, reducedChunk.getStatus()); } } + private WorkChunk freshFetchWorkChunk(String chunkId) { + return runInTransaction(() -> + myWorkChunkRepository.findById(chunkId) + .map(e-> JobInstanceUtil.fromEntityToWorkChunk(e)) + .orElseThrow(IllegalArgumentException::new)); + } + + @Nonnull private JobInstance createInstance() { JobInstance instance = new JobInstance(); @@ -722,15 +688,11 @@ private String storeJobInstanceAndUpdateWithEndTime(StatusEnum theStatus, int mi final String id = mySvc.storeNewInstance(jobInstance); - jobInstance.setInstanceId(id); - final LocalDateTime localDateTime = LocalDateTime.now() - .minusMinutes(minutes); - ourLog.info("localDateTime: {}", localDateTime); - jobInstance.setEndTime(Date.from(localDateTime - .atZone(ZoneId.systemDefault()) - .toInstant())); + mySvc.updateInstance(id, instance->{ + instance.setEndTime(Date.from(Instant.now().minus(minutes, ChronoUnit.MINUTES))); + return true; + }); - mySvc.updateInstance(jobInstance); return id; } @@ -740,11 +702,11 @@ private String storeJobInstanceAndUpdateWithEndTime(StatusEnum theStatus, int mi */ public static List provideStatuses() { return List.of( - Arguments.of(StatusEnum.QUEUED, true), - Arguments.of(StatusEnum.IN_PROGRESS, true), - Arguments.of(StatusEnum.ERRORED, true), - Arguments.of(StatusEnum.FAILED, false), - Arguments.of(StatusEnum.COMPLETED, false) + Arguments.of(WorkChunkStatusEnum.QUEUED, true), + Arguments.of(WorkChunkStatusEnum.IN_PROGRESS, true), + Arguments.of(WorkChunkStatusEnum.ERRORED, true), + Arguments.of(WorkChunkStatusEnum.FAILED, false), + Arguments.of(WorkChunkStatusEnum.COMPLETED, false) ); } } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseIT.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseIT.java index 80c7ca3b1479..66a07486a823 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseIT.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/BulkExportUseCaseIT.java @@ -15,6 +15,7 @@ import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository; import ca.uhn.fhir.jpa.entity.Batch2JobInstanceEntity; import ca.uhn.fhir.jpa.entity.Batch2WorkChunkEntity; +import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; import ca.uhn.fhir.jpa.searchparam.SearchParameterMap; import ca.uhn.fhir.jpa.test.Batch2JobHelper; @@ -30,6 +31,7 @@ import com.google.common.collect.Sets; import org.apache.commons.io.Charsets; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -40,6 +42,7 @@ import org.hl7.fhir.r4.model.Coverage; import org.hl7.fhir.r4.model.Encounter; import org.hl7.fhir.r4.model.Enumerations; +import org.hl7.fhir.r4.model.Extension; import org.hl7.fhir.r4.model.Group; import org.hl7.fhir.r4.model.IdType; import org.hl7.fhir.r4.model.InstantType; @@ -102,6 +105,49 @@ public class BulkExportUseCaseIT extends BaseResourceProviderR4Test { @Nested public class SpecConformanceIT { + + @Test + public void testBulkExportJobsAreMetaTaggedWithJobIdAndExportId() throws IOException { + //Given a patient exists + Patient p = new Patient(); + p.setId("Pat-1"); + myClient.update().resource(p).execute(); + + //And Given we start a bulk export job with a specific export id + String pollingLocation = submitBulkExportForTypesWithExportId("im-an-export-identifier", "Patient"); + String jobId = getJobIdFromPollingLocation(pollingLocation); + myBatch2JobHelper.awaitJobCompletion(jobId); + + //Then: When the poll shows as complete, all attributes should be filled. + HttpGet statusGet = new HttpGet(pollingLocation); + String expectedOriginalUrl = myClient.getServerBase() + "/$export?_type=Patient&_exportId=im-an-export-identifier"; + try (CloseableHttpResponse status = ourHttpClient.execute(statusGet)) { + assertEquals(200, status.getStatusLine().getStatusCode()); + String responseContent = IOUtils.toString(status.getEntity().getContent(), StandardCharsets.UTF_8); + assertTrue(isNotBlank(responseContent), responseContent); + + ourLog.info(responseContent); + + BulkExportResponseJson result = JsonUtil.deserialize(responseContent, BulkExportResponseJson.class); + assertThat(result.getRequest(), is(equalTo(expectedOriginalUrl))); + assertThat(result.getOutput(), is(not(empty()))); + String binary_url = result.getOutput().get(0).getUrl(); + Binary binaryResource = myClient.read().resource(Binary.class).withUrl(binary_url).execute(); + + List extension = binaryResource.getMeta().getExtension(); + assertThat(extension, hasSize(3)); + + assertThat(extension.get(0).getUrl(), is(equalTo(JpaConstants.BULK_META_EXTENSION_EXPORT_IDENTIFIER))); + assertThat(extension.get(0).getValue().toString(), is(equalTo("im-an-export-identifier"))); + + assertThat(extension.get(1).getUrl(), is(equalTo(JpaConstants.BULK_META_EXTENSION_JOB_ID))); + assertThat(extension.get(1).getValue().toString(), is(equalTo(jobId))); + + assertThat(extension.get(2).getUrl(), is(equalTo(JpaConstants.BULK_META_EXTENSION_RESOURCE_TYPE))); + assertThat(extension.get(2).getValue().toString(), is(equalTo("Patient"))); + } + } + @Test public void testBatchJobsAreOnlyReusedIfInProgress() throws IOException { //Given a patient exists @@ -110,7 +156,7 @@ public void testBatchJobsAreOnlyReusedIfInProgress() throws IOException { myClient.update().resource(p).execute(); //And Given we start a bulk export job - String pollingLocation = submitBulkExportForTypes("Patient"); + String pollingLocation = submitBulkExportForTypesWithExportId("my-export-id-","Patient"); String jobId = getJobIdFromPollingLocation(pollingLocation); myBatch2JobHelper.awaitJobCompletion(jobId); @@ -285,8 +331,16 @@ public void export_shouldNotExportBinaryResource_whenTypeParameterOmitted() thro } private String submitBulkExportForTypes(String... theTypes) throws IOException { + return submitBulkExportForTypesWithExportId(null, theTypes); + } + private String submitBulkExportForTypesWithExportId(String theExportId, String... theTypes) throws IOException { String typeString = String.join(",", theTypes); - HttpGet httpGet = new HttpGet(myClient.getServerBase() + "/$export?_type=" + typeString); + String uri = myClient.getServerBase() + "/$export?_type=" + typeString; + if (!StringUtils.isBlank(theExportId)) { + uri += "&_exportId=" + theExportId; + } + + HttpGet httpGet = new HttpGet(uri); httpGet.addHeader(Constants.HEADER_PREFER, Constants.HEADER_PREFER_RESPOND_ASYNC); String pollingLocation; try (CloseableHttpResponse status = ourHttpClient.execute(httpGet)) { diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt/svc/BulkDataImportR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt/svc/BulkDataImportR4Test.java index a4ab9daf4261..c7c7bb69969a 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt/svc/BulkDataImportR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/bulk/imprt/svc/BulkDataImportR4Test.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.test.utilities.ITestDataBuilder; import ca.uhn.fhir.util.BundleBuilder; +import org.hamcrest.Matchers; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.r4.model.Patient; import org.junit.jupiter.api.AfterEach; @@ -105,7 +106,7 @@ public void afterResetInterceptors() { private void setupRetryFailures() { myWorkChannel.addInterceptor(new ExecutorChannelInterceptor() { @Override - public void afterMessageHandled(Message message, MessageChannel channel, MessageHandler handler, Exception ex) { + public void afterMessageHandled(@Nonnull Message message, @Nonnull MessageChannel channel, @Nonnull MessageHandler handler, Exception ex) { if (ex != null) { ourLog.info("Work channel received exception {}", ex.getMessage()); channel.send(message); @@ -145,8 +146,7 @@ public void testFlow_ErrorDuringWrite() { failed.add(StatusEnum.ERRORED); assertTrue(failed.contains(instance.getStatus()), instance.getStatus() + " is the actual status"); String errorMsg = instance.getErrorMessage(); - assertTrue(errorMsg.contains("Too many errors"), errorMsg); - assertTrue(errorMsg.contains("Too many errors"), MyFailAfterThreeCreatesInterceptor.ERROR_MESSAGE); + assertThat(errorMsg, Matchers.containsString("Too many errors")); } finally { myWorkChannel.clearInterceptorsForUnitTest(); } @@ -173,7 +173,7 @@ public void testFlow_TransactionRows() { assertNotNull(instance); assertEquals(StatusEnum.COMPLETED, instance.getStatus()); - IBundleProvider searchResults = myPatientDao.search(SearchParameterMap.newSynchronous()); + IBundleProvider searchResults = myPatientDao.search(SearchParameterMap.newSynchronous(), mySrd); assertEquals(transactionsPerFile * fileCount, searchResults.sizeOrThrowNpe()); } @@ -250,7 +250,7 @@ public void testJobsAreRegisteredWithJobRegistry() { } @Interceptor - public class MyFailAfterThreeCreatesInterceptor { + public static class MyFailAfterThreeCreatesInterceptor { public static final String ERROR_MESSAGE = "This is an error message"; diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/BinaryStorageInterceptorR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/BinaryStorageInterceptorR4Test.java index 4a533d89c485..0a50dd75bb2d 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/BinaryStorageInterceptorR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/BinaryStorageInterceptorR4Test.java @@ -1,18 +1,27 @@ package ca.uhn.fhir.jpa.provider.r4; import ca.uhn.fhir.i18n.Msg; +import ca.uhn.fhir.interceptor.api.Hook; +import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc; import ca.uhn.fhir.jpa.binary.interceptor.BinaryStorageInterceptor; import ca.uhn.fhir.jpa.binstore.MemoryBinaryStorageSvcImpl; import ca.uhn.fhir.jpa.provider.BaseResourceProviderR4Test; +import ca.uhn.fhir.rest.client.api.IClientInterceptor; +import ca.uhn.fhir.rest.client.api.IHttpRequest; +import ca.uhn.fhir.rest.client.api.IHttpResponse; +import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.util.HapiExtensions; +import org.hl7.fhir.instance.model.api.IBaseHasExtensions; +import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.r4.model.Binary; import org.hl7.fhir.r4.model.DocumentReference; import org.hl7.fhir.r4.model.Enumerations; +import org.hl7.fhir.r4.model.Extension; import org.hl7.fhir.r4.model.StringType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -21,6 +30,10 @@ import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import java.util.stream.Collectors; + +import java.io.IOException; + import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.MatcherAssert.assertThat; @@ -70,6 +83,42 @@ public void after() throws Exception { myInterceptorRegistry.unregisterInterceptor(myBinaryStorageInterceptor); } + class BinaryFilePrefixingInterceptor{ + + @Hook(Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX) + public String provideFilenameForBinary(RequestDetails theRequestDetails, IBaseResource theResource) { + ourLog.info("Received binary for prefixing!" + theResource.getIdElement()); + String extensionValus = ((IBaseHasExtensions) theResource.getMeta()).getExtension().stream().map(ext -> ext.getValue().toString()).collect(Collectors.joining("-")); + return "prefix-" + extensionValus + "-"; + } + } + @Test + public void testCreatingExternalizedBinaryTriggersPointcut() { + BinaryFilePrefixingInterceptor interceptor = new BinaryFilePrefixingInterceptor(); + myInterceptorRegistry.registerInterceptor(interceptor); + // Create a resource with two metadata extensions on the binary + Binary binary = new Binary(); + binary.setContentType("application/octet-stream"); + Extension ext = binary.getMeta().addExtension(); + ext.setUrl("http://foo"); + ext.setValue(new StringType("bar")); + + Extension ext2 = binary.getMeta().addExtension(); + ext2.setUrl("http://foo2"); + ext2.setValue(new StringType("bar2")); + + binary.setData(SOME_BYTES); + DaoMethodOutcome outcome = myBinaryDao.create(binary, mySrd); + + // Make sure it was externalized + IIdType id = outcome.getId().toUnqualifiedVersionless(); + String encoded = myFhirContext.newJsonParser().setPrettyPrint(true).encodeResourceToString(outcome.getResource()); + ourLog.info("Encoded: {}", encoded); + assertThat(encoded, containsString(HapiExtensions.EXT_EXTERNALIZED_BINARY_ID)); + assertThat(encoded, (containsString("prefix-bar-bar2-"))); + myInterceptorRegistry.unregisterInterceptor(interceptor); + } + @Test public void testCreateAndRetrieveBinary_ServerAssignedId_ExternalizedBinary() { @@ -90,7 +139,6 @@ public void testCreateAndRetrieveBinary_ServerAssignedId_ExternalizedBinary() { Binary output = myBinaryDao.read(id, mySrd); assertEquals("application/octet-stream", output.getContentType()); assertArrayEquals(SOME_BYTES, output.getData()); - } diff --git a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java index 53541ac7953b..1c0ead9d30f5 100644 --- a/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java +++ b/hapi-fhir-jpaserver-test-r4/src/test/java/ca/uhn/fhir/jpa/provider/r4/MultitenantBatchOperationR4Test.java @@ -8,6 +8,7 @@ import ca.uhn.fhir.interceptor.model.RequestPartitionId; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.delete.job.ReindexTestHelper; +import ca.uhn.fhir.jpa.model.entity.ResourceIndexedSearchParamToken; import ca.uhn.fhir.rest.api.CacheControlDirective; import ca.uhn.fhir.rest.api.server.RequestDetails; import ca.uhn.fhir.rest.server.provider.ProviderConstants; @@ -27,10 +28,13 @@ import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import static ca.uhn.fhir.jpa.model.util.JpaConstants.DEFAULT_PARTITION_NAME; +import static org.awaitility.Awaitility.await; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.in; import static org.hamcrest.Matchers.isA; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -156,11 +160,22 @@ public void testReindexEverything() { myBatch2JobHelper.awaitJobCompletion(jobId.getValue()); + logAllTokenIndexes(); + // validate + runInTransaction(()->{ + long indexedSps = myResourceIndexedSearchParamTokenDao + .findAll() + .stream() + .filter(t->t.getParamName().equals("alleleName")) + .count(); + assertEquals(1, indexedSps, ()->"Token indexes:\n * " + myResourceIndexedSearchParamTokenDao.findAll().stream().filter(t->t.getParamName().equals("alleleName")).map(ResourceIndexedSearchParamToken::toString).collect(Collectors.joining("\n * "))); + }); + List alleleObservationIds = reindexTestHelper.getAlleleObservationIds(myClient); // Only the one in the first tenant should be indexed myTenantClientInterceptor.setTenantId(TENANT_A); - MatcherAssert.assertThat(reindexTestHelper.getAlleleObservationIds(myClient), hasSize(1)); + await().until(() -> reindexTestHelper.getAlleleObservationIds(myClient), hasSize(1)); assertEquals(obsFinalA.getIdPart(), alleleObservationIds.get(0)); myTenantClientInterceptor.setTenantId(TENANT_B); MatcherAssert.assertThat(reindexTestHelper.getAlleleObservationIds(myClient), hasSize(0)); @@ -180,9 +195,17 @@ public void testReindexEverything() { myBatch2JobHelper.awaitJobCompletion(jobId.getValue()); + runInTransaction(()->{ + long indexedSps = myResourceIndexedSearchParamTokenDao + .findAll() + .stream() + .filter(t->t.getParamName().equals("alleleName")) + .count(); + assertEquals(3, indexedSps, ()->"Token indexes:\n * " + myResourceIndexedSearchParamTokenDao.findAll().stream().filter(t->t.getParamName().equals("alleleName")).map(ResourceIndexedSearchParamToken::toString).collect(Collectors.joining("\n * "))); + }); myTenantClientInterceptor.setTenantId(DEFAULT_PARTITION_NAME); - MatcherAssert.assertThat(reindexTestHelper.getAlleleObservationIds(myClient), hasSize(1)); + await().until(() -> reindexTestHelper.getAlleleObservationIds(myClient), hasSize(1)); } @Test diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java index 791fa9ead96a..70e7bd898471 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/BaseJpaR4Test.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.jpa.test; - /*- * #%L * HAPI FHIR JPA Server Test Utilities @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.jpa.test; import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.context.support.IValidationSupport; diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java index e8baf8285595..b6ca41f343b4 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/Batch2JobHelper.java @@ -100,7 +100,7 @@ public JobInstance awaitJobHasStatus(String theBatchJobId, int theSecondsToWait, .map(t -> t.getJobDefinitionId() + "/" + t.getStatus().name()) .collect(Collectors.joining("\n")); String currentStatus = myJobCoordinator.getInstance(theBatchJobId).getStatus().name(); - fail("Job still has status " + currentStatus + " - All statuses:\n" + statuses); + fail("Job " + theBatchJobId + " still has status " + currentStatus + " - All statuses:\n" + statuses); } return myJobCoordinator.getInstance(theBatchJobId); } @@ -128,12 +128,13 @@ private boolean checkStatusWithMaintenancePass(String theBatchJobId, StatusEnum. return true; } myJobMaintenanceService.runMaintenancePass(); - Thread.sleep(1000); return hasStatus(theBatchJobId, theExpectedStatuses); } private boolean hasStatus(String theBatchJobId, StatusEnum[] theExpectedStatuses) { - return ArrayUtils.contains(theExpectedStatuses, getStatus(theBatchJobId)); + StatusEnum status = getStatus(theBatchJobId); + ourLog.debug("Checking status of {} in {}: is {}", theBatchJobId, theExpectedStatuses, status); + return ArrayUtils.contains(theExpectedStatuses, status); } private StatusEnum getStatus(String theBatchJobId) { diff --git a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java index 3e8142334179..f1d23e798c62 100644 --- a/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java +++ b/hapi-fhir-jpaserver-test-utilities/src/main/java/ca/uhn/fhir/jpa/test/config/TestR4Config.java @@ -92,6 +92,7 @@ public class TestR4Config { ourMaxThreads = 100; } } + ourLog.warn("ourMaxThreads={}", ourMaxThreads); } private final Deque myLastStackTrace = new LinkedList<>(); diff --git a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/bulk/BulkDataExportOptions.java b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/bulk/BulkDataExportOptions.java index dabeff215527..51fa3b5b0038 100644 --- a/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/bulk/BulkDataExportOptions.java +++ b/hapi-fhir-server/src/main/java/ca/uhn/fhir/rest/api/server/bulk/BulkDataExportOptions.java @@ -33,6 +33,7 @@ // They don't seem to serve any distinct purpose so they should be collapsed into 1 public class BulkDataExportOptions { + public enum ExportStyle { PATIENT, GROUP, @@ -49,6 +50,8 @@ public enum ExportStyle { private IIdType myGroupId; private Set myPatientIds; + private String myExportIdentifier; + public void setOutputFormat(String theOutputFormat) { myOutputFormat = theOutputFormat; } @@ -132,4 +135,12 @@ public Set getPatientIds() { public void setPatientIds(Set thePatientIds) { myPatientIds = thePatientIds; } + + public String getExportIdentifier() { + return myExportIdentifier; + } + + public void setExportIdentifier(String theExportIdentifier) { + myExportIdentifier = theExportIdentifier; + } } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidator.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidator.java index 5cbc699b929c..90a534b30c55 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidator.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidator.java @@ -23,15 +23,20 @@ import ca.uhn.fhir.batch2.api.IJobParametersValidator; import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc; import ca.uhn.fhir.jpa.bulk.export.provider.BulkDataExportProvider; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; +import org.apache.commons.lang3.StringUtils; +import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import org.springframework.beans.factory.annotation.Autowired; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.util.ArrayList; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class BulkExportJobParametersValidator implements IJobParametersValidator { @@ -41,6 +46,9 @@ public class BulkExportJobParametersValidator implements IJobParametersValidator @Autowired private DaoRegistry myDaoRegistry; + @Autowired + private IBinaryStorageSvc myBinaryStorageSvc; + @Nullable @Override public List validate(@Nonnull BulkExportJobParameters theParameters) { @@ -62,6 +70,13 @@ public List validate(@Nonnull BulkExportJobParameters theParameters) { if (!Constants.CT_FHIR_NDJSON.equalsIgnoreCase(theParameters.getOutputFormat())) { errorMsgs.add("The only allowed format for Bulk Export is currently " + Constants.CT_FHIR_NDJSON); } + // validate the exportId + if (!StringUtils.isBlank(theParameters.getExportIdentifier())) { + + if (!myBinaryStorageSvc.isValidBlobId(theParameters.getExportIdentifier())) { + errorMsgs.add("Export ID does not conform to the current blob storage implementation's limitations."); + } + } // validate for group BulkDataExportOptions.ExportStyle style = theParameters.getExportStyle(); @@ -84,4 +99,5 @@ public List validate(@Nonnull BulkExportJobParameters theParameters) { return errorMsgs; } + } diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java index a0e10eb31f34..4138d7d57c1e 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/WriteBinaryStep.java @@ -35,9 +35,13 @@ import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao; import ca.uhn.fhir.jpa.api.model.DaoMethodOutcome; import ca.uhn.fhir.rest.api.server.SystemRequestDetails; +import ca.uhn.fhir.jpa.model.util.JpaConstants; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.util.BinaryUtil; +import org.apache.commons.lang3.StringUtils; import org.hl7.fhir.instance.model.api.IBaseBinary; +import org.hl7.fhir.instance.model.api.IBaseExtension; +import org.hl7.fhir.instance.model.api.IBaseHasExtensions; import org.hl7.fhir.instance.model.api.IIdType; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; @@ -74,6 +78,8 @@ public RunOutcome run(@Nonnull StepExecutionDetails theStepExecutionDetails, ExpandedResourcesList expandedResources, IBaseBinary binary) { + // Note that this applies only to hl7.org structures, so these extensions will not be added + // to DSTU2 structures + if (binary.getMeta() instanceof IBaseHasExtensions) { + IBaseHasExtensions meta = (IBaseHasExtensions) binary.getMeta(); + + //export identifier, potentially null. + String exportIdentifier = theStepExecutionDetails.getParameters().getExportIdentifier(); + if (!StringUtils.isBlank(exportIdentifier)) { + IBaseExtension exportIdentifierExtension = meta.addExtension(); + exportIdentifierExtension.setUrl(JpaConstants.BULK_META_EXTENSION_EXPORT_IDENTIFIER); + exportIdentifierExtension.setValue(myFhirContext.newPrimitiveString(exportIdentifier)); + } + + //job id + IBaseExtension jobExtension = meta.addExtension(); + jobExtension.setUrl(JpaConstants.BULK_META_EXTENSION_JOB_ID); + jobExtension.setValue(myFhirContext.newPrimitiveString(theStepExecutionDetails.getInstance().getInstanceId())); + + //resource type + IBaseExtension typeExtension = meta.addExtension(); + typeExtension.setUrl(JpaConstants.BULK_META_EXTENSION_RESOURCE_TYPE); + typeExtension.setValue(myFhirContext.newPrimitiveString(expandedResources.getResourceType())); + } else { + ourLog.warn("Could not attach metadata extensions to binary resource, as this binary metadata does not support extensions"); + } + } + /** * Returns an output stream writer * (exposed for testing) diff --git a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportJobParameters.java b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportJobParameters.java index 4f59960e8aa7..d56562271f47 100644 --- a/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportJobParameters.java +++ b/hapi-fhir-storage-batch2-jobs/src/main/java/ca/uhn/fhir/batch2/jobs/export/models/BulkExportJobParameters.java @@ -47,6 +47,9 @@ public class BulkExportJobParameters extends BulkExportJobBase { @JsonProperty("since") private Date myStartDate; + @JsonProperty("exportId") + private String myExportId; + @JsonProperty("filters") private List myFilters; @@ -75,10 +78,35 @@ public class BulkExportJobParameters extends BulkExportJobBase { @JsonProperty("expandMdm") private boolean myExpandMdm; + + public static BulkExportJobParameters createFromExportJobParameters(BulkExportParameters theParameters) { + BulkExportJobParameters params = new BulkExportJobParameters(); + params.setResourceTypes(theParameters.getResourceTypes()); + params.setExportStyle(theParameters.getExportStyle()); + params.setExportIdentifier(theParameters.getExportIdentifier()); + params.setFilters(theParameters.getFilters()); + params.setPostFetchFilterUrls(theParameters.getPostFetchFilterUrls()); + params.setGroupId(theParameters.getGroupId()); + params.setOutputFormat(theParameters.getOutputFormat()); + params.setStartDate(theParameters.getStartDate()); + params.setExpandMdm(theParameters.isExpandMdm()); + params.setPatientIds(theParameters.getPatientIds()); + params.setOriginalRequestUrl(theParameters.getOriginalRequestUrl()); + return params; + } + + public String getExportIdentifier() { + return myExportId; + } + public List getResourceTypes() { return myResourceTypes; } + public void setExportIdentifier(String theExportId) { + myExportId = theExportId; + } + public void setResourceTypes(List theResourceTypes) { myResourceTypes = theResourceTypes; } @@ -158,19 +186,4 @@ public String getOriginalRequestUrl() { return myOriginalRequestUrl; } - public static BulkExportJobParameters createFromExportJobParameters(BulkExportParameters theParameters) { - BulkExportJobParameters params = new BulkExportJobParameters(); - params.setResourceTypes(theParameters.getResourceTypes()); - params.setExportStyle(theParameters.getExportStyle()); - params.setFilters(theParameters.getFilters()); - params.setPostFetchFilterUrls(theParameters.getPostFetchFilterUrls()); - params.setGroupId(theParameters.getGroupId()); - params.setOutputFormat(theParameters.getOutputFormat()); - params.setStartDate(theParameters.getStartDate()); - params.setExpandMdm(theParameters.isExpandMdm()); - params.setPatientIds(theParameters.getPatientIds()); - params.setOriginalRequestUrl(theParameters.getOriginalRequestUrl()); - return params; - } - } diff --git a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidatorTest.java b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidatorTest.java index 9da414afa05c..f2bd5482de39 100644 --- a/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidatorTest.java +++ b/hapi-fhir-storage-batch2-jobs/src/test/java/ca/uhn/fhir/batch2/jobs/export/BulkExportJobParametersValidatorTest.java @@ -2,6 +2,7 @@ import ca.uhn.fhir.batch2.jobs.export.models.BulkExportJobParameters; import ca.uhn.fhir.jpa.api.dao.DaoRegistry; +import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc; import ca.uhn.fhir.rest.api.Constants; import ca.uhn.fhir.rest.api.server.bulk.BulkDataExportOptions; import org.junit.jupiter.api.Test; @@ -18,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.when; @@ -27,6 +29,9 @@ public class BulkExportJobParametersValidatorTest { @Mock private DaoRegistry myDaoRegistry; + @Mock + private IBinaryStorageSvc myIBinaryStorageSvc; + @InjectMocks private BulkExportJobParametersValidator myValidator; @@ -55,6 +60,38 @@ public void validate_validParametersForSystem_returnsEmptyList() { assertTrue(result.isEmpty()); } + + @Test + public void validate_exportId_illegal_characters() { + BulkExportJobParameters parameters = createSystemExportParameters(); + parameters.setExportIdentifier("exportId&&&"); + // when + when(myDaoRegistry.isResourceTypeSupported(anyString())) + .thenReturn(true); + when(myIBinaryStorageSvc.isValidBlobId(any())).thenReturn(false); + List errors = myValidator.validate(parameters); + + // verify + assertNotNull(errors); + assertEquals(1, errors.size()); + assertEquals(errors.get(0), "Export ID does not conform to the current blob storage implementation's limitations."); + } + + @Test + public void validate_exportId_legal_characters() { + BulkExportJobParameters parameters = createSystemExportParameters(); + parameters.setExportIdentifier("HELLO!/WORLD/"); + // when + when(myDaoRegistry.isResourceTypeSupported(anyString())) + .thenReturn(true); + + when(myIBinaryStorageSvc.isValidBlobId(any())).thenReturn(true); + List errors = myValidator.validate(parameters); + + // verify + assertNotNull(errors); + assertEquals(0, errors.size()); + } @Test public void validate_validParametersForPatient_returnsEmptyList() { // setup diff --git a/hapi-fhir-storage-batch2-test-utilities/pom.xml b/hapi-fhir-storage-batch2-test-utilities/pom.xml new file mode 100644 index 000000000000..8a72aec67bf8 --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/pom.xml @@ -0,0 +1,74 @@ + + + 4.0.0 + + + ca.uhn.hapi.fhir + hapi-deployable-pom + 6.4.5 + ../hapi-deployable-pom/pom.xml + + + hapi-fhir-storage-batch2-test-utilities + + HAPI FHIR JPA Server - Batch2 specification tests + Batch2 is a framework for managing and executing long running "batch" jobs + + + + ca.uhn.hapi.fhir + hapi-fhir-storage-batch2 + ${project.version} + + + ca.uhn.hapi.fhir + hapi-fhir-test-utilities + ${project.version} + + + + + org.hamcrest + hamcrest + compile + + + org.junit.jupiter + junit-jupiter + compile + + + org.junit.jupiter + junit-jupiter-api + compile + + + org.junit.jupiter + junit-jupiter-engine + compile + + + org.junit.jupiter + junit-jupiter-params + compile + + + org.hamcrest + hamcrest + compile + + + org.mockito + mockito-core + compile + + + org.mockito + mockito-junit-jupiter + compile + + + + diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java new file mode 100644 index 000000000000..1f1928a4d777 --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/AbstractIJobPersistenceSpecificationTest.java @@ -0,0 +1,699 @@ +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 specification tests + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +package ca.uhn.hapi.fhir.batch2.test; + +import ca.uhn.fhir.batch2.api.IJobPersistence; +import ca.uhn.fhir.batch2.api.RunOutcome; +import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; +import ca.uhn.fhir.batch2.maintenance.JobChunkProgressAccumulator; +import ca.uhn.fhir.batch2.maintenance.JobInstanceProcessor; +import ca.uhn.fhir.batch2.model.JobDefinition; +import ca.uhn.fhir.batch2.model.JobInstance; +import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.util.StopWatch; +import ca.uhn.hapi.fhir.batch2.test.support.TestJobParameters; +import ca.uhn.hapi.fhir.batch2.test.support.TestJobStep2InputType; +import ca.uhn.hapi.fhir.batch2.test.support.TestJobStep3InputType; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.transaction.PlatformTransactionManager; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.TransactionStatus; +import org.springframework.transaction.support.TransactionCallbackWithoutResult; +import org.springframework.transaction.support.TransactionTemplate; + +import javax.annotation.Nonnull; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.stream.Collectors; + +import static org.awaitility.Awaitility.await; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.empty; +import static org.hamcrest.Matchers.emptyString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + + +/** + * Specification tests for batch2 storage and event system. + * These tests are abstract, and do not depend on JPA. + * Test setups should use the public batch2 api to create scenarios. + */ +public abstract class AbstractIJobPersistenceSpecificationTest { + private static final Logger ourLog = LoggerFactory.getLogger(AbstractIJobPersistenceSpecificationTest.class); + + public static final String JOB_DEFINITION_ID = "definition-id"; + public static final String TARGET_STEP_ID = "step-id"; + public static final String DEF_CHUNK_ID = "definition-chunkId"; + public static final String STEP_CHUNK_ID = "step-chunkId"; + public static final int JOB_DEF_VER = 1; + public static final int SEQUENCE_NUMBER = 1; + public static final String CHUNK_DATA = "{\"key\":\"value\"}"; + public static final String ERROR_MESSAGE_A = "This is an error message: A"; + public static final String ERROR_MESSAGE_B = "This is a different error message: B"; + public static final String ERROR_MESSAGE_C = "This is a different error message: C"; + + @Autowired + private IJobPersistence mySvc; + + @Nested + class WorkChunkStorage { + + @Test + public void testStoreAndFetchWorkChunk_NoData() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + + String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, null); + + WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); + assertNull(chunk.getData()); + } + + @Test + public void testStoreAndFetchWorkChunk_WithData() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + + String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, 0, CHUNK_DATA); + assertNotNull(id); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, freshFetchWorkChunk(id).getStatus())); + + WorkChunk chunk = mySvc.onWorkChunkDequeue(id).orElseThrow(IllegalArgumentException::new); + assertEquals(36, chunk.getInstanceId().length()); + assertEquals(JOB_DEFINITION_ID, chunk.getJobDefinitionId()); + assertEquals(JOB_DEF_VER, chunk.getJobDefinitionVersion()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(CHUNK_DATA, chunk.getData()); + + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, freshFetchWorkChunk(id).getStatus())); + } + + /** + * Should match the diagram in batch2_states.md + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ + @Nested + class StateTransitions { + + private String myInstanceId; + private String myChunkId; + + @BeforeEach + void setUp() { + JobInstance jobInstance = createInstance(); + myInstanceId = mySvc.storeNewInstance(jobInstance); + + } + + private String createChunk() { + return storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, myInstanceId, 0, CHUNK_DATA); + } + + @Test + public void chunkCreation_isQueued() { + + myChunkId = createChunk(); + + WorkChunk fetchedWorkChunk = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.QUEUED, fetchedWorkChunk.getStatus(), "New chunks are QUEUED"); + } + + @Test + public void chunkReceived_queuedToInProgress() { + + myChunkId = createChunk(); + + // the worker has received the chunk, and marks it started. + WorkChunk chunk = mySvc.onWorkChunkDequeue(myChunkId).orElseThrow(IllegalArgumentException::new); + + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + assertEquals(CHUNK_DATA, chunk.getData()); + + // verify the db was updated too + WorkChunk fetchedWorkChunk = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, fetchedWorkChunk.getStatus()); + } + + @Nested + class InProgressActions { + @BeforeEach + void setUp() { + // setup - the worker has received the chunk, and has marked it IN_PROGRESS. + myChunkId = createChunk(); + mySvc.onWorkChunkDequeue(myChunkId); + } + + @Test + public void processingOk_inProgressToSuccess_clearsDataSavesRecordCount() { + + // execution ok + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(myChunkId, 3, 0)); + + // verify the db was updated + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.COMPLETED, workChunkEntity.getStatus()); + assertNull(workChunkEntity.getData()); + assertEquals(3, workChunkEntity.getRecordsProcessed()); + assertNull(workChunkEntity.getErrorMessage()); + assertEquals(0, workChunkEntity.getErrorCount()); + } + + @Test + public void processingRetryableError_inProgressToError_bumpsCountRecordsMessage() { + + // execution had a retryable error + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, ERROR_MESSAGE_A)); + + // verify the db was updated + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, workChunkEntity.getStatus()); + assertEquals(ERROR_MESSAGE_A, workChunkEntity.getErrorMessage()); + assertEquals(1, workChunkEntity.getErrorCount()); + } + + @Test + public void processingFailure_inProgressToFailed() { + + // execution had a failure + mySvc.onWorkChunkFailed(myChunkId, "some error"); + + // verify the db was updated + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.FAILED, workChunkEntity.getStatus()); + assertEquals("some error", workChunkEntity.getErrorMessage()); + } + } + + @Nested + class ErrorActions { + public static final String FIRST_ERROR_MESSAGE = ERROR_MESSAGE_A; + @BeforeEach + void setUp() { + // setup - the worker has received the chunk, and has marked it IN_PROGRESS. + myChunkId = createChunk(); + mySvc.onWorkChunkDequeue(myChunkId); + // execution had a retryable error + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, FIRST_ERROR_MESSAGE)); + } + + /** + * The consumer will retry after a retryable error is thrown + */ + @Test + void errorRetry_errorToInProgress() { + + // when consumer restarts chunk + WorkChunk chunk = mySvc.onWorkChunkDequeue(myChunkId).orElseThrow(IllegalArgumentException::new); + + // then + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + + // verify the db state, error message, and error count + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, workChunkEntity.getStatus()); + assertEquals(FIRST_ERROR_MESSAGE, workChunkEntity.getErrorMessage(), "Original error message kept"); + assertEquals(1, workChunkEntity.getErrorCount(), "error count kept"); + } + + @Test + void errorRetry_repeatError_increasesErrorCount() { + // setup - the consumer is re-trying, and marks it IN_PROGRESS + mySvc.onWorkChunkDequeue(myChunkId); + + + // when another error happens + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, ERROR_MESSAGE_B)); + + + // verify the state, new message, and error count + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, workChunkEntity.getStatus()); + assertEquals(ERROR_MESSAGE_B, workChunkEntity.getErrorMessage(), "new error message"); + assertEquals(2, workChunkEntity.getErrorCount(), "error count inc"); + } + + @Test + void errorThenRetryAndComplete_addsErrorCounts() { + // setup - the consumer is re-trying, and marks it IN_PROGRESS + mySvc.onWorkChunkDequeue(myChunkId); + + // then it completes ok. + mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(myChunkId, 3, 1)); + + // verify the state, new message, and error count + var workChunkEntity = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.COMPLETED, workChunkEntity.getStatus()); + assertEquals(FIRST_ERROR_MESSAGE, workChunkEntity.getErrorMessage(), "Error message kept."); + assertEquals(2, workChunkEntity.getErrorCount(), "error combined with earlier error"); + } + + @Test + void errorRetry_maxErrors_movesToFailed() { + // we start with 1 error already + + // 2nd try + mySvc.onWorkChunkDequeue(myChunkId); + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, ERROR_MESSAGE_B)); + var chunk = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, chunk.getStatus()); + assertEquals(2, chunk.getErrorCount()); + + // 3rd try + mySvc.onWorkChunkDequeue(myChunkId); + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, ERROR_MESSAGE_B)); + chunk = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, chunk.getStatus()); + assertEquals(3, chunk.getErrorCount()); + + // 4th try + mySvc.onWorkChunkDequeue(myChunkId); + mySvc.onWorkChunkError(new WorkChunkErrorEvent(myChunkId, ERROR_MESSAGE_C)); + chunk = freshFetchWorkChunk(myChunkId); + assertEquals(WorkChunkStatusEnum.FAILED, chunk.getStatus()); + assertEquals(4, chunk.getErrorCount()); + assertThat("Error message contains last error", chunk.getErrorMessage(), containsString(ERROR_MESSAGE_C)); + assertThat("Error message contains error count and complaint", chunk.getErrorMessage(), containsString("many errors: 4")); + } + } + } + + @Test + public void testFetchChunks() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + + List ids = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + String id = storeWorkChunk(JOB_DEFINITION_ID, TARGET_STEP_ID, instanceId, i, CHUNK_DATA); + ids.add(id); + } + + List chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 3, 0); + assertNull(chunks.get(0).getData()); + assertNull(chunks.get(1).getData()); + assertNull(chunks.get(2).getData()); + assertThat(chunks.stream().map(WorkChunk::getId).collect(Collectors.toList()), + contains(ids.get(0), ids.get(1), ids.get(2))); + + chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 3, 1); + assertThat(chunks.stream().map(WorkChunk::getId).collect(Collectors.toList()), + contains(ids.get(3), ids.get(4), ids.get(5))); + + chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 3, 2); + assertThat(chunks.stream().map(WorkChunk::getId).collect(Collectors.toList()), + contains(ids.get(6), ids.get(7), ids.get(8))); + + chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 3, 3); + assertThat(chunks.stream().map(WorkChunk::getId).collect(Collectors.toList()), + contains(ids.get(9))); + + chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 3, 4); + assertThat(chunks.stream().map(WorkChunk::getId).collect(Collectors.toList()), + empty()); + } + + + @Test + public void testMarkChunkAsCompleted_Success() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, CHUNK_DATA); + assertNotNull(chunkId); + + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, freshFetchWorkChunk(chunkId).getStatus())); + + sleepUntilTimeChanges(); + + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); + assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + assertNotNull(chunk.getCreateTime()); + assertNotNull(chunk.getStartTime()); + assertNull(chunk.getEndTime()); + assertNull(chunk.getRecordsProcessed()); + assertNotNull(chunk.getData()); + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.IN_PROGRESS, freshFetchWorkChunk(chunkId).getStatus())); + + sleepUntilTimeChanges(); + + runInTransaction(() -> mySvc.onWorkChunkCompletion(new WorkChunkCompletionEvent(chunkId, 50, 0))); + + WorkChunk entity = freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.COMPLETED, entity.getStatus()); + assertEquals(50, entity.getRecordsProcessed()); + assertNotNull(entity.getCreateTime()); + assertNotNull(entity.getStartTime()); + assertNotNull(entity.getEndTime()); + assertNull(entity.getData()); + assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); + assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); + } + + + @Test + public void testMarkChunkAsCompleted_Error() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); + assertNotNull(chunkId); + + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, freshFetchWorkChunk(chunkId).getStatus())); + + sleepUntilTimeChanges(); + + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); + assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + + sleepUntilTimeChanges(); + + WorkChunkErrorEvent request = new WorkChunkErrorEvent(chunkId, ERROR_MESSAGE_A); + mySvc.onWorkChunkError(request); + runInTransaction(() -> { + WorkChunk entity = freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); + assertEquals(ERROR_MESSAGE_A, entity.getErrorMessage()); + assertNotNull(entity.getCreateTime()); + assertNotNull(entity.getStartTime()); + assertNotNull(entity.getEndTime()); + assertEquals(1, entity.getErrorCount()); + assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); + assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); + }); + + // Mark errored again + + WorkChunkErrorEvent request2 = new WorkChunkErrorEvent(chunkId, "This is an error message 2"); + mySvc.onWorkChunkError(request2); + runInTransaction(() -> { + WorkChunk entity = freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.ERRORED, entity.getStatus()); + assertEquals("This is an error message 2", entity.getErrorMessage()); + assertNotNull(entity.getCreateTime()); + assertNotNull(entity.getStartTime()); + assertNotNull(entity.getEndTime()); + assertEquals(2, entity.getErrorCount()); + assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); + assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); + }); + + List chunks = mySvc.fetchWorkChunksWithoutData(instanceId, 100, 0); + assertEquals(1, chunks.size()); + assertEquals(2, chunks.get(0).getErrorCount()); + } + + @Test + public void testMarkChunkAsCompleted_Fail() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + String chunkId = storeWorkChunk(DEF_CHUNK_ID, STEP_CHUNK_ID, instanceId, SEQUENCE_NUMBER, null); + assertNotNull(chunkId); + + runInTransaction(() -> assertEquals(WorkChunkStatusEnum.QUEUED, freshFetchWorkChunk(chunkId).getStatus())); + + sleepUntilTimeChanges(); + + WorkChunk chunk = mySvc.onWorkChunkDequeue(chunkId).orElseThrow(IllegalArgumentException::new); + assertEquals(SEQUENCE_NUMBER, chunk.getSequence()); + assertEquals(WorkChunkStatusEnum.IN_PROGRESS, chunk.getStatus()); + + sleepUntilTimeChanges(); + + mySvc.onWorkChunkFailed(chunkId, "This is an error message"); + runInTransaction(() -> { + WorkChunk entity = freshFetchWorkChunk(chunkId); + assertEquals(WorkChunkStatusEnum.FAILED, entity.getStatus()); + assertEquals("This is an error message", entity.getErrorMessage()); + assertNotNull(entity.getCreateTime()); + assertNotNull(entity.getStartTime()); + assertNotNull(entity.getEndTime()); + assertTrue(entity.getCreateTime().getTime() < entity.getStartTime().getTime()); + assertTrue(entity.getStartTime().getTime() < entity.getEndTime().getTime()); + }); + } + + @Test + public void markWorkChunksWithStatusAndWipeData_marksMultipleChunksWithStatus_asExpected() { + JobInstance instance = createInstance(); + String instanceId = mySvc.storeNewInstance(instance); + ArrayList chunkIds = new ArrayList<>(); + for (int i = 0; i < 10; i++) { + WorkChunkCreateEvent chunk = new WorkChunkCreateEvent( + "defId", + 1, + "stepId", + instanceId, + 0, + "{}" + ); + String id = mySvc.onWorkChunkCreate(chunk); + chunkIds.add(id); + } + + runInTransaction(() -> mySvc.markWorkChunksWithStatusAndWipeData(instance.getInstanceId(), chunkIds, WorkChunkStatusEnum.COMPLETED, null)); + + Iterator reducedChunks = mySvc.fetchAllWorkChunksIterator(instanceId, true); + + while (reducedChunks.hasNext()) { + WorkChunk reducedChunk = reducedChunks.next(); + assertTrue(chunkIds.contains(reducedChunk.getId())); + assertEquals(WorkChunkStatusEnum.COMPLETED, reducedChunk.getStatus()); + } + } + } + + /** + * Test + * * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ + @Nested + class InstanceStateTransitions { + + @Test + void createInstance_createsInQueuedWithChunk() { + // given + JobDefinition jd = withJobDefinition(); + + // when + IJobPersistence.CreateResult createResult = + newTxTemplate().execute(status-> + mySvc.onCreateWithFirstChunk(jd, "{}")); + + // then + ourLog.info("job and chunk created {}", createResult); + assertNotNull(createResult); + assertThat(createResult.jobInstanceId, not(emptyString())); + assertThat(createResult.workChunkId, not(emptyString())); + + JobInstance jobInstance = freshFetchJobInstance(createResult.jobInstanceId); + assertThat(jobInstance.getStatus(), equalTo(StatusEnum.QUEUED)); + assertThat(jobInstance.getParameters(), equalTo("{}")); + + WorkChunk firstChunk = freshFetchWorkChunk(createResult.workChunkId); + assertThat(firstChunk.getStatus(), equalTo(WorkChunkStatusEnum.QUEUED)); + assertNull(firstChunk.getData(), "First chunk data is null - only uses parameters"); + } + + @Test + void testCreateInstance_firstChunkDequeued_movesToInProgress() { + // given + JobDefinition jd = withJobDefinition(); + IJobPersistence.CreateResult createResult = newTxTemplate().execute(status-> + mySvc.onCreateWithFirstChunk(jd, "{}")); + assertNotNull(createResult); + + // when + newTxTemplate().execute(status -> mySvc.onChunkDequeued(createResult.jobInstanceId)); + + // then + JobInstance jobInstance = freshFetchJobInstance(createResult.jobInstanceId); + assertThat(jobInstance.getStatus(), equalTo(StatusEnum.IN_PROGRESS)); + } + + + + @ParameterizedTest + @EnumSource(StatusEnum.class) + void cancelRequest_cancelsJob_whenNotFinalState(StatusEnum theState) { + // given + JobInstance cancelledInstance = createInstance(); + cancelledInstance.setStatus(theState); + String instanceId1 = mySvc.storeNewInstance(cancelledInstance); + mySvc.cancelInstance(instanceId1); + + JobInstance normalInstance = createInstance(); + normalInstance.setStatus(theState); + String instanceId2 = mySvc.storeNewInstance(normalInstance); + + JobDefinitionRegistry jobDefinitionRegistry = new JobDefinitionRegistry(); + jobDefinitionRegistry.addJobDefinitionIfNotRegistered(withJobDefinition()); + + + // when + runInTransaction(()-> new JobInstanceProcessor(mySvc, null, instanceId1, new JobChunkProgressAccumulator(), null, jobDefinitionRegistry) + .process()); + + + // then + JobInstance freshInstance1 = mySvc.fetchInstance(instanceId1).orElseThrow(); + if (theState.isCancellable()) { + assertEquals(StatusEnum.CANCELLED, freshInstance1.getStatus(), "cancel request processed"); + assertThat(freshInstance1.getErrorMessage(), containsString("Job instance cancelled")); + } else { + assertEquals(theState, freshInstance1.getStatus(), "cancel request ignored - state unchanged"); + assertNull(freshInstance1.getErrorMessage(), "no error message"); + } + JobInstance freshInstance2 = mySvc.fetchInstance(instanceId2).orElseThrow(); + assertEquals(theState, freshInstance2.getStatus(), "cancel request ignored - cancelled not set"); + } + } + + @Test + void testInstanceUpdate_modifierApplied() { + // given + String instanceId = mySvc.storeNewInstance(createInstance()); + + // when + mySvc.updateInstance(instanceId, instance ->{ + instance.setErrorCount(42); + return true; + }); + + // then + JobInstance jobInstance = freshFetchJobInstance(instanceId); + assertEquals(42, jobInstance.getErrorCount()); + } + + @Test + void testInstanceUpdate_modifierNotAppliedWhenPredicateReturnsFalse() { + // given + JobInstance instance1 = createInstance(); + boolean initialValue = true; + instance1.setFastTracking(initialValue); + String instanceId = mySvc.storeNewInstance(instance1); + + // when + mySvc.updateInstance(instanceId, instance ->{ + instance.setFastTracking(false); + return false; + }); + + // then + JobInstance jobInstance = freshFetchJobInstance(instanceId); + assertEquals(initialValue, jobInstance.isFastTracking()); + } + + private JobDefinition withJobDefinition() { + return JobDefinition.newBuilder() + .setJobDefinitionId(JOB_DEFINITION_ID) + .setJobDefinitionVersion(JOB_DEF_VER) + .setJobDescription("A job description") + .setParametersType(TestJobParameters.class) + .addFirstStep(TARGET_STEP_ID, "the first step", TestJobStep2InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addIntermediateStep("2nd-step-id", "the second step", TestJobStep3InputType.class, (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .addLastStep("last-step-id", "the final step", (theStepExecutionDetails, theDataSink) -> new RunOutcome(0)) + .build(); + } + + + @Nonnull + private JobInstance createInstance() { + JobInstance instance = new JobInstance(); + instance.setJobDefinitionId(JOB_DEFINITION_ID); + instance.setStatus(StatusEnum.QUEUED); + instance.setJobDefinitionVersion(JOB_DEF_VER); + instance.setParameters(CHUNK_DATA); + instance.setReport("TEST"); + return instance; + } + + private String storeWorkChunk(String theJobDefinitionId, String theTargetStepId, String theInstanceId, int theSequence, String theSerializedData) { + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(theJobDefinitionId, JOB_DEF_VER, theTargetStepId, theInstanceId, theSequence, theSerializedData); + return mySvc.onWorkChunkCreate(batchWorkChunk); + } + + + protected abstract PlatformTransactionManager getTxManager(); + protected abstract WorkChunk freshFetchWorkChunk(String theChunkId); + protected JobInstance freshFetchJobInstance(String theInstanceId) { + return runInTransaction(() -> mySvc.fetchInstance(theInstanceId).orElseThrow()); + } + + public TransactionTemplate newTxTemplate() { + TransactionTemplate retVal = new TransactionTemplate(getTxManager()); + retVal.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + retVal.afterPropertiesSet(); + return retVal; + } + + public void runInTransaction(Runnable theRunnable) { + newTxTemplate().execute(new TransactionCallbackWithoutResult() { + @Override + protected void doInTransactionWithoutResult(@Nonnull TransactionStatus theStatus) { + theRunnable.run(); + } + }); + } + + public T runInTransaction(Callable theRunnable) { + return newTxTemplate().execute(t -> { + try { + return theRunnable.call(); + } catch (Exception theE) { + throw new InternalErrorException(theE); + } + }); + } + + + /** + * Sleep until at least 1 ms has elapsed + */ + public void sleepUntilTimeChanges() { + StopWatch sw = new StopWatch(); + await().until(() -> sw.getMillis() > 0); + } + + + +} diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobParameters.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobParameters.java new file mode 100644 index 000000000000..2c4c3e43b48f --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobParameters.java @@ -0,0 +1,72 @@ +package ca.uhn.hapi.fhir.batch2.test.support; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 specification tests + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.model.api.annotation.PasswordField; +import com.fasterxml.jackson.annotation.JsonProperty; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotBlank; + +public class TestJobParameters implements IModelJson { + + @JsonProperty("param1") + @NotBlank + private String myParam1; + + @JsonProperty("param2") + @NotBlank + @Length(min = 5, max = 100) + private String myParam2; + + @JsonProperty(value = "password") + @PasswordField + private String myPassword; + + public String getPassword() { + return myPassword; + } + + public TestJobParameters setPassword(String thePassword) { + myPassword = thePassword; + return this; + } + + public String getParam1() { + return myParam1; + } + + public TestJobParameters setParam1(String theParam1) { + myParam1 = theParam1; + return this; + } + + public String getParam2() { + return myParam2; + } + + public TestJobParameters setParam2(String theParam2) { + myParam2 = theParam2; + return this; + } + +} diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep2InputType.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep2InputType.java new file mode 100644 index 000000000000..5a3421e148c9 --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep2InputType.java @@ -0,0 +1,63 @@ +package ca.uhn.hapi.fhir.batch2.test.support; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 specification tests + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.model.api.IModelJson; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class TestJobStep2InputType implements IModelJson { + + /** + * Constructor + */ + public TestJobStep2InputType() { + } + + /** + * Constructor + */ + public TestJobStep2InputType(String theData1, String theData2) { + myData1 = theData1; + myData2 = theData2; + } + + @JsonProperty("data1") + private String myData1; + @JsonProperty("data2") + private String myData2; + + public String getData1() { + return myData1; + } + + public void setData1(String theData1) { + myData1 = theData1; + } + + public String getData2() { + return myData2; + } + + public void setData2(String theData2) { + myData2 = theData2; + } + +} diff --git a/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep3InputType.java b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep3InputType.java new file mode 100644 index 000000000000..47cebd8a98db --- /dev/null +++ b/hapi-fhir-storage-batch2-test-utilities/src/main/java/ca/uhn/hapi/fhir/batch2/test/support/TestJobStep3InputType.java @@ -0,0 +1,51 @@ +package ca.uhn.hapi.fhir.batch2.test.support; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 specification tests + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import ca.uhn.fhir.model.api.IModelJson; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class TestJobStep3InputType implements IModelJson { + + @JsonProperty("data3") + private String myData3; + @JsonProperty("data4") + private String myData4; + + public String getData3() { + return myData3; + } + + public TestJobStep3InputType setData3(String theData1) { + myData3 = theData1; + return this; + } + + public String getData4() { + return myData4; + } + + public TestJobStep3InputType setData4(String theData2) { + myData4 = theData2; + return this; + } + +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java index 6eca71408b64..73d2a93e1ee5 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IJobPersistence.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.api; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,20 +17,26 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.api; -import ca.uhn.fhir.batch2.coordinator.BatchWorkChunk; import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; +import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.i18n.Msg; +import org.apache.commons.lang3.builder.ToStringBuilder; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.data.domain.Page; import org.springframework.data.domain.Pageable; import org.springframework.transaction.annotation.Propagation; import org.springframework.transaction.annotation.Transactional; +import javax.annotation.Nonnull; +import java.util.Collections; import java.util.Date; import java.util.Iterator; import java.util.List; @@ -40,27 +44,15 @@ import java.util.Set; import java.util.stream.Stream; -public interface IJobPersistence { - - /** - * Stores a chunk of work for later retrieval. This method should be atomic and should only - * return when the chunk has been successfully stored in the database. - *

- * Chunk should be stored with a status of {@link ca.uhn.fhir.batch2.model.StatusEnum#QUEUED} - * - * @param theBatchWorkChunk the batch work chunk to be stored - * @return a globally unique identifier for this chunk. This should be a sequentially generated ID, a UUID, or something like that which is guaranteed to never overlap across jobs or instances. - */ - String storeWorkChunk(BatchWorkChunk theBatchWorkChunk); - - /** - * Fetches a chunk of work from storage, and update the stored status to {@link StatusEnum#IN_PROGRESS}. - * This will only fetch chunks which are currently QUEUED or ERRORRED. - * - * @param theChunkId The ID, as returned by {@link #storeWorkChunk(BatchWorkChunk theBatchWorkChunk)} - * @return The chunk of work - */ - Optional fetchWorkChunkSetStartTimeAndMarkInProgress(String theChunkId); +/** + * + * Some of this is tested in {@link ca.uhn.hapi.fhir.batch2.test.AbstractIJobPersistenceSpecificationTest} + * This is a transactional interface, but we have pushed the declaration of calls that have + * {@code @Transactional(propagation = Propagation.REQUIRES_NEW)} down to the implementations since we have a synchronized + * wrapper that was double-createing the NEW transaction. + */ +public interface IJobPersistence extends IWorkChunkPersistence { + Logger ourLog = LoggerFactory.getLogger(IJobPersistence.class); /** @@ -68,6 +60,7 @@ public interface IJobPersistence { * * @param theInstance The details */ + @Transactional(propagation = Propagation.REQUIRED) String storeNewInstance(JobInstance theInstance); /** @@ -83,7 +76,6 @@ default List fetchInstances(String theJobDefinitionId, Set fetchInstances(FetchJobInstancesRequest theRequest, int theStart, int theBatchSize); @@ -101,10 +93,6 @@ default List fetchInstances(String theJobDefinitionId, Set fetchInstancesByJobDefinitionId(String theJobDefinitionId, int theCount, int theStart); @@ -117,69 +105,13 @@ default Page fetchJobInstances(JobInstanceFetchRequest theRequest) return Page.empty(); } - /** - * Marks a given chunk as having errored (i.e. may be recoverable) - * - * @param theChunkId The chunk ID - */ - @Deprecated - void markWorkChunkAsErroredAndIncrementErrorCount(String theChunkId, String theErrorMessage); - /** - * Marks a given chunk as having errored (ie, may be recoverable) - * - * Returns the work chunk. - * - * NB: For backwards compatibility reasons, it could be an empty optional, but - * this doesn't mean it has no workchunk (just implementers are not updated) - * - * @param theParameters - the parameters for marking the workchunk with error - * @return - workchunk optional, if available. - */ - default Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWorkChunkAsErrorRequest theParameters) { - // old method - please override me - markWorkChunkAsErroredAndIncrementErrorCount(theParameters.getChunkId(), theParameters.getErrorMsg()); - return Optional.empty(); // returning empty so as not to break implementers - } - - /** - * Marks a given chunk as having failed (i.e. probably not recoverable) - * - * @param theChunkId The chunk ID - */ - void markWorkChunkAsFailed(String theChunkId, String theErrorMessage); - - /** - * Marks a given chunk as having finished - * - * @param theChunkId The chunk ID - * @param theRecordsProcessed The number of records completed during chunk processing - */ - void markWorkChunkAsCompletedAndClearData(String theInstanceId, String theChunkId, int theRecordsProcessed); - - /** - * Marks all work chunks with the provided status and erases the data - * @param theInstanceId - the instance id - * @param theChunkIds - the ids of work chunks being reduced to single chunk - * @param theStatus - the status to mark - * @param theErrorMsg - error message (if status warrants it) - */ - void markWorkChunksWithStatusAndWipeData(String theInstanceId, List theChunkIds, StatusEnum theStatus, String theErrorMsg); - - /** - * Increments the work chunk error count by the given amount - * - * @param theChunkId The chunk ID - * @param theIncrementBy The number to increment the error count by - */ - void incrementWorkChunkErrorCount(String theChunkId, int theIncrementBy); - - @Transactional(propagation = Propagation.REQUIRES_NEW) boolean canAdvanceInstanceToNextStep(String theInstanceId, String theCurrentStepId); /** * Fetches all chunks for a given instance, without loading the data * + * TODO MB this seems to only be used by tests. Can we use the iterator instead? * @param theInstanceId The instance ID * @param thePageSize The page size * @param thePageIndex The page index @@ -197,32 +129,39 @@ default Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWor Iterator fetchAllWorkChunksIterator(String theInstanceId, boolean theWithData); /** - * Deprecated, use {@link ca.uhn.fhir.batch2.api.IJobPersistence#fetchAllWorkChunksForStepStream(String, String)} - * Fetch all chunks with data for a given instance for a given step id - * @param theInstanceId - * @param theStepId - * @return - an iterator for fetching work chunks + * Fetch all chunks with data for a given instance for a given step id - read-only. + * + * @return - a stream for fetching work chunks */ - @Deprecated - Iterator fetchAllWorkChunksForStepIterator(String theInstanceId, String theStepId); - + @Transactional(propagation = Propagation.MANDATORY, readOnly = true) + Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); /** - * Fetch all chunks with data for a given instance for a given step id - * @param theInstanceId - * @param theStepId - * @return - a stream for fetching work chunks + * Callback to update a JobInstance within a locked transaction. + * Return true from the callback if the record write should continue, or false if + * the change should be discarded. */ - Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); + @FunctionalInterface + interface JobInstanceUpdateCallback { + /** + * Modify theInstance within a write-lock transaction. + * @param theInstance a copy of the instance to modify. + * @return true if the change to theInstance should be written back to the db. + */ + boolean doUpdate(JobInstance theInstance); + } /** - * Update the stored instance. If the status is changing, use {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater} - * instead to ensure state-change callbacks are invoked properly. + * Goofy hack for now to create a tx boundary. + * If the status is changing, use {@link ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater} + * instead to ensure state-change callbacks are invoked properly. * - * @param theInstance The instance - Must contain an ID - * @return true if the status changed + * @param theInstanceId the id of the instance to modify + * @param theModifier a hook to modify the instance - return true to finish the record write + * @return true if the instance was modified */ - boolean updateInstance(JobInstance theInstance); + // todo mb consider changing callers to actual objects we can unit test. + boolean updateInstance(String theInstanceId, JobInstanceUpdateCallback theModifier); /** * Deletes the instance and all associated work chunks @@ -246,9 +185,11 @@ default Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWor */ boolean markInstanceAsCompleted(String theInstanceId); - @Transactional(propagation = Propagation.REQUIRES_NEW) boolean markInstanceAsStatus(String theInstance, StatusEnum theStatusEnum); + @Transactional(propagation = Propagation.MANDATORY) + boolean markInstanceAsStatusWhenStatusIn(String theInstance, StatusEnum theStatusEnum, Set thePriorStates); + /** * Marks an instance as cancelled * @@ -256,7 +197,61 @@ default Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWor */ JobOperationResultJson cancelInstance(String theInstanceId); - List fetchallchunkidsforstepWithStatus(String theInstanceId, String theStepId, StatusEnum theStatusEnum); - void updateInstanceUpdateTime(String theInstanceId); + + + + /* + * State transition events for job instances. + * These cause the transitions along {@link ca.uhn.fhir.batch2.model.StatusEnum} + * + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ + /////// + // job events + + class CreateResult { + public final String jobInstanceId; + public final String workChunkId; + + public CreateResult(String theJobInstanceId, String theWorkChunkId) { + jobInstanceId = theJobInstanceId; + workChunkId = theWorkChunkId; + } + + @Override + public String toString() { + return new ToStringBuilder(this) + .append("jobInstanceId", jobInstanceId) + .append("workChunkId", workChunkId) + .toString(); + } + } + + @Nonnull + default CreateResult onCreateWithFirstChunk(JobDefinition theJobDefinition, String theParameters) { + JobInstance instance = JobInstance.fromJobDefinition(theJobDefinition); + instance.setParameters(theParameters); + instance.setStatus(StatusEnum.QUEUED); + + String instanceId = storeNewInstance(instance); + ourLog.info("Stored new {} job {} with status {}", theJobDefinition.getJobDefinitionId(), instanceId, instance.getStatus()); + ourLog.debug("Job parameters: {}", instance.getParameters()); + + WorkChunkCreateEvent batchWorkChunk = WorkChunkCreateEvent.firstChunk(theJobDefinition, instanceId); + String chunkId = onWorkChunkCreate(batchWorkChunk); + return new CreateResult(instanceId, chunkId); + + } + + /** + * Move from QUEUED->IN_PROGRESS when a work chunk arrives. + * Ignore other prior states. + * @return did the transition happen + */ + default boolean onChunkDequeued(String theJobInstanceId) { + return markInstanceAsStatusWhenStatusIn(theJobInstanceId, StatusEnum.IN_PROGRESS, Collections.singleton(StatusEnum.QUEUED)); + } + void processCancelRequests(); + } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java new file mode 100644 index 000000000000..4836a79cfdf8 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/api/IWorkChunkPersistence.java @@ -0,0 +1,149 @@ +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ +package ca.uhn.fhir.batch2.api; + +import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import org.springframework.transaction.annotation.Propagation; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +/** + * Work Chunk api, implementing the WorkChunk state machine. + * Test specification is in {@link ca.uhn.hapi.fhir.batch2.test.AbstractIJobPersistenceSpecificationTest} + * + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ +public interface IWorkChunkPersistence { + + ////////////////////////////////// + // WorkChunk calls + ////////////////////////////////// + + /** + * Stores a chunk of work for later retrieval. + * The first state event, as the chunk is created. + * This method should be atomic and should only + * return when the chunk has been successfully stored in the database. + * Chunk should be stored with a status of {@link WorkChunkStatusEnum#QUEUED} + * + * @param theBatchWorkChunk the batch work chunk to be stored + * @return a globally unique identifier for this chunk. + */ + @Transactional(propagation = Propagation.REQUIRED) + String onWorkChunkCreate(WorkChunkCreateEvent theBatchWorkChunk); + + /** + * On arrival at a worker. + * The second state event, as the worker starts processing. + * Transition to {@link WorkChunkStatusEnum#IN_PROGRESS} if unless not in QUEUED or ERRORRED state. + * + * @param theChunkId The ID from {@link #onWorkChunkCreate} + * @return The WorkChunk or empty if no chunk exists, or not in a runnable state (QUEUED or ERRORRED) + */ + @Transactional(propagation = Propagation.REQUIRED) + Optional onWorkChunkDequeue(String theChunkId); + + /** + * A retryable error. + * Transition to {@link WorkChunkStatusEnum#ERRORED} unless max-retries passed, then + * transition to {@link WorkChunkStatusEnum#FAILED}. + * + * @param theParameters - the error message and max retry count. + * @return - the new status - ERRORED or ERRORED, depending on retry count + */ + WorkChunkStatusEnum onWorkChunkError(WorkChunkErrorEvent theParameters); + + /** + * An unrecoverable error. + * Transition to {@link WorkChunkStatusEnum#FAILED} + * + * @param theChunkId The chunk ID + */ + @Transactional(propagation = Propagation.REQUIRED) + void onWorkChunkFailed(String theChunkId, String theErrorMessage); + + + /** + * Report success and complete the chunk. + * Transition to {@link WorkChunkStatusEnum#COMPLETED} + * + * @param theEvent with record and error count + */ + @Transactional(propagation = Propagation.REQUIRED) + void onWorkChunkCompletion(WorkChunkCompletionEvent theEvent); + + /** + * Marks all work chunks with the provided status and erases the data + * + * @param theInstanceId - the instance id + * @param theChunkIds - the ids of work chunks being reduced to single chunk + * @param theStatus - the status to mark + * @param theErrorMsg - error message (if status warrants it) + */ + @Transactional(propagation = Propagation.MANDATORY) + void markWorkChunksWithStatusAndWipeData(String theInstanceId, List theChunkIds, WorkChunkStatusEnum theStatus, String theErrorMsg); + + + /** + * Fetches all chunks for a given instance, without loading the data + * + * @param theInstanceId The instance ID + * @param thePageSize The page size + * @param thePageIndex The page index + */ + List fetchWorkChunksWithoutData(String theInstanceId, int thePageSize, int thePageIndex); + + + /** + * Fetch all chunks for a given instance. + * + * @param theInstanceId - instance id + * @param theWithData - whether or not to include the data + * @return - an iterator for fetching work chunks + */ + Iterator fetchAllWorkChunksIterator(String theInstanceId, boolean theWithData); + + + /** + * Fetch all chunks with data for a given instance for a given step id + * + * @return - a stream for fetching work chunks + */ + Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId); + + /** + * Fetch chunk ids for starting a gated step. + * + * @param theInstanceId the job + * @param theStepId the step that is starting + * @return the WorkChunk ids + */ + List fetchAllChunkIdsForStepWithStatus(String theInstanceId, String theStepId, WorkChunkStatusEnum theStatusEnum); + + +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java index 7614ae24741c..bf9c04918f28 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/config/BaseBatch2Config.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.config; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.config; import ca.uhn.fhir.batch2.api.IJobCoordinator; import ca.uhn.fhir.batch2.api.IJobMaintenanceService; @@ -59,8 +58,8 @@ public JobDefinitionRegistry batch2JobDefinitionRegistry() { } @Bean - public WorkChunkProcessor jobStepExecutorService(BatchJobSender theBatchJobSender, IHapiTransactionService theTransactionService) { - return new WorkChunkProcessor(myPersistence, theBatchJobSender, theTransactionService); + public WorkChunkProcessor jobStepExecutorService(BatchJobSender theBatchJobSender) { + return new WorkChunkProcessor(myPersistence, theBatchJobSender); } @Bean @@ -72,14 +71,16 @@ public BatchJobSender batchJobSender() { public IJobCoordinator batch2JobCoordinator(JobDefinitionRegistry theJobDefinitionRegistry, BatchJobSender theBatchJobSender, WorkChunkProcessor theExecutor, - IJobMaintenanceService theJobMaintenanceService) { + IJobMaintenanceService theJobMaintenanceService, + IHapiTransactionService theTransactionService) { return new JobCoordinatorImpl( theBatchJobSender, batch2ProcessingChannelReceiver(myChannelFactory), myPersistence, theJobDefinitionRegistry, theExecutor, - theJobMaintenanceService); + theJobMaintenanceService, + theTransactionService); } @Bean diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java index fc6b8d270c4d..48f91e06bd0d 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImpl.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.coordinator; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.coordinator; import ca.uhn.fhir.batch2.api.IJobCoordinator; import ca.uhn.fhir.batch2.api.IJobMaintenanceService; @@ -34,6 +33,7 @@ import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; +import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelReceiver; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; @@ -48,7 +48,6 @@ import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; import java.util.Arrays; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -65,6 +64,7 @@ public class JobCoordinatorImpl implements IJobCoordinator { private final MessageHandler myReceiverHandler; private final JobQuerySvc myJobQuerySvc; private final JobParameterJsonValidator myJobParameterJsonValidator; + private final IHapiTransactionService myTransactionService; /** * Constructor @@ -74,7 +74,8 @@ public JobCoordinatorImpl(@Nonnull BatchJobSender theBatchJobSender, @Nonnull IJobPersistence theJobPersistence, @Nonnull JobDefinitionRegistry theJobDefinitionRegistry, @Nonnull WorkChunkProcessor theExecutorSvc, - @Nonnull IJobMaintenanceService theJobMaintenanceService) { + @Nonnull IJobMaintenanceService theJobMaintenanceService, + @Nonnull IHapiTransactionService theTransactionService) { Validate.notNull(theJobPersistence); myJobPersistence = theJobPersistence; @@ -82,16 +83,14 @@ public JobCoordinatorImpl(@Nonnull BatchJobSender theBatchJobSender, myWorkChannelReceiver = theWorkChannelReceiver; myJobDefinitionRegistry = theJobDefinitionRegistry; - myReceiverHandler = new WorkChannelMessageHandler(theJobPersistence, theJobDefinitionRegistry, theBatchJobSender, theExecutorSvc, theJobMaintenanceService); + myReceiverHandler = new WorkChannelMessageHandler(theJobPersistence, theJobDefinitionRegistry, theBatchJobSender, theExecutorSvc, theJobMaintenanceService, theTransactionService); myJobQuerySvc = new JobQuerySvc(theJobPersistence, theJobDefinitionRegistry); myJobParameterJsonValidator = new JobParameterJsonValidator(); + myTransactionService = theTransactionService; } @Override public Batch2JobStartResponse startInstance(JobInstanceStartRequest theStartRequest) { - JobDefinition jobDefinition = myJobDefinitionRegistry - .getLatestJobDefinition(theStartRequest.getJobDefinitionId()).orElseThrow(() -> new IllegalArgumentException(Msg.code(2063) + "Unknown job definition ID: " + theStartRequest.getJobDefinitionId())); - String paramsString = theStartRequest.getParameters(); if (isBlank(paramsString)) { throw new InvalidRequestException(Msg.code(2065) + "No parameters supplied"); @@ -103,9 +102,9 @@ public Batch2JobStartResponse startInstance(JobInstanceStartRequest theStartRequ List existing = myJobPersistence.fetchInstances(request, 0, 1000); if (!existing.isEmpty()) { // we'll look for completed ones first... otherwise, take any of the others - Collections.sort(existing, (o1, o2) -> -(o1.getStatus().ordinal() - o2.getStatus().ordinal())); + existing.sort((o1, o2) -> -(o1.getStatus().ordinal() - o2.getStatus().ordinal())); - JobInstance first = existing.stream().findFirst().get(); + JobInstance first = existing.stream().findFirst().orElseThrow(); Batch2JobStartResponse response = new Batch2JobStartResponse(); response.setJobId(first.getInstanceId()); @@ -117,24 +116,21 @@ public Batch2JobStartResponse startInstance(JobInstanceStartRequest theStartRequ } } - myJobParameterJsonValidator.validateJobParameters(theStartRequest, jobDefinition); + JobDefinition jobDefinition = myJobDefinitionRegistry + .getLatestJobDefinition(theStartRequest.getJobDefinitionId()).orElseThrow(() -> new IllegalArgumentException(Msg.code(2063) + "Unknown job definition ID: " + theStartRequest.getJobDefinitionId())); - JobInstance instance = JobInstance.fromJobDefinition(jobDefinition); - instance.setParameters(theStartRequest.getParameters()); - instance.setStatus(StatusEnum.QUEUED); + myJobParameterJsonValidator.validateJobParameters(theStartRequest, jobDefinition); - String instanceId = myJobPersistence.storeNewInstance(instance); - ourLog.info("Stored new {} job {} with status {}", jobDefinition.getJobDefinitionId(), instanceId, instance.getStatus()); - ourLog.debug("Job parameters: {}", instance.getParameters()); - BatchWorkChunk batchWorkChunk = BatchWorkChunk.firstChunk(jobDefinition, instanceId); - String chunkId = myJobPersistence.storeWorkChunk(batchWorkChunk); + IJobPersistence.CreateResult instanceAndFirstChunk = + myTransactionService.withRequest(null).execute(() -> + myJobPersistence.onCreateWithFirstChunk(jobDefinition, theStartRequest.getParameters())); - JobWorkNotification workNotification = JobWorkNotification.firstStepNotification(jobDefinition, instanceId, chunkId); + JobWorkNotification workNotification = JobWorkNotification.firstStepNotification(jobDefinition, instanceAndFirstChunk.jobInstanceId, instanceAndFirstChunk.workChunkId); myBatchJobSender.sendWorkChannelMessage(workNotification); Batch2JobStartResponse response = new Batch2JobStartResponse(); - response.setJobId(instanceId); + response.setJobId(instanceAndFirstChunk.jobInstanceId); return response; } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java index 43877e4613c1..79307cfae86a 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDataSink.java @@ -26,6 +26,7 @@ import ca.uhn.fhir.batch2.model.JobDefinitionStep; import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.JobWorkNotification; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; import ca.uhn.fhir.batch2.model.WorkChunkData; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.util.Logs; @@ -72,8 +73,8 @@ public void accept(WorkChunkData theData) { OT dataValue = theData.getData(); String dataValueString = JsonUtil.serialize(dataValue, false); - BatchWorkChunk batchWorkChunk = new BatchWorkChunk(myJobDefinitionId, myJobDefinitionVersion, targetStepId, instanceId, sequence, dataValueString); - String chunkId = myJobPersistence.storeWorkChunk(batchWorkChunk); + WorkChunkCreateEvent batchWorkChunk = new WorkChunkCreateEvent(myJobDefinitionId, myJobDefinitionVersion, targetStepId, instanceId, sequence, dataValueString); + String chunkId = myJobPersistence.onWorkChunkCreate(batchWorkChunk); myLastChunkId.set(chunkId); if (!myGatedExecution) { diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDefinitionRegistry.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDefinitionRegistry.java index 609b6dd0db1f..7db1554c8766 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDefinitionRegistry.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobDefinitionRegistry.java @@ -26,8 +26,8 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.context.ConfigurationException; import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.util.Logs; import com.google.common.collect.ImmutableSortedMap; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; @@ -47,6 +47,7 @@ public class JobDefinitionRegistry { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); + // TODO MB is this safe? Can ue use ConcurrentHashMap instead? private volatile Map>> myJobs = new HashMap<>(); /** @@ -165,13 +166,9 @@ public boolean isEmpty() { return myJobs.isEmpty(); } - public Optional> getJobDefinition(JobInstance theJobInstance) { - return getJobDefinition(theJobInstance.getJobDefinitionId(), theJobInstance.getJobDefinitionVersion()); - } - @SuppressWarnings("unchecked") - public JobDefinition getJobDefinitionOrThrowException(JobInstance theJobInstance) { - return (JobDefinition) getJobDefinitionOrThrowException(theJobInstance.getJobDefinitionId(), theJobInstance.getJobDefinitionVersion()); + public JobDefinition getJobDefinitionOrThrowException(JobInstance theJobInstance) { + return (JobDefinition) getJobDefinitionOrThrowException(theJobInstance.getJobDefinitionId(), theJobInstance.getJobDefinitionVersion()); } public Collection getJobDefinitionVersions(String theDefinitionId) { diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java index e0bae4cdd16b..28a19a51f9a6 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutor.java @@ -22,10 +22,10 @@ import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.api.IJobPersistence; -import ca.uhn.fhir.batch2.channel.BatchJobSender; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; +import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater; import ca.uhn.fhir.model.api.IModelJson; @@ -39,7 +39,6 @@ public class JobStepExecutor myCursor; JobStepExecutor(@Nonnull IJobPersistence theJobPersistence, - @Nonnull BatchJobSender theBatchJobSender, @Nonnull JobInstance theInstance, WorkChunk theWorkChunk, @Nonnull JobWorkCursor theCursor, @@ -59,7 +57,6 @@ public class JobStepExecutor stepExecutorOutput = myJobExecutorSvc.doExecution( myCursor, @@ -84,8 +80,11 @@ public void executeStep() { if (stepExecutorOutput.getDataSink().firstStepProducedNothing()) { ourLog.info("First step of job myInstance {} produced no work chunks, marking as completed and setting end date", myInstanceId); - myInstance.setEndTime(new Date()); - myJobInstanceStatusUpdater.setCompleted(myInstance); + myJobPersistence.updateInstance(myInstance.getInstanceId(), instance->{ + instance.setEndTime(new Date()); + myJobInstanceStatusUpdater.updateInstanceStatus(instance, StatusEnum.COMPLETED); + return true; + }); } if (myInstance.isFastTracking()) { @@ -98,13 +97,17 @@ private void handleFastTracking(BaseDataSink theDataSink) { ourLog.debug("Gated job {} step {} produced exactly one chunk: Triggering a maintenance pass.", myDefinition.getJobDefinitionId(), myCursor.currentStep.getStepId()); boolean success = myJobMaintenanceService.triggerMaintenancePass(); if (!success) { - myInstance.setFastTracking(false); - myJobPersistence.updateInstance(myInstance); + myJobPersistence.updateInstance(myInstance.getInstanceId(), instance-> { + instance.setFastTracking(false); + return true; + }); } } else { ourLog.debug("Gated job {} step {} produced {} chunks: Disabling fast tracking.", myDefinition.getJobDefinitionId(), myCursor.currentStep.getStepId(), theDataSink.getWorkChunkCount()); - myInstance.setFastTracking(false); - myJobPersistence.updateInstance(myInstance); + myJobPersistence.updateInstance(myInstance.getInstanceId(), instance-> { + instance.setFastTracking(false); + return true; + }); } } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutorFactory.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutorFactory.java index d6cef77975c5..6d60be1b1598 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutorFactory.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/JobStepExecutorFactory.java @@ -50,6 +50,6 @@ public JobStepExecutorFactory(@Nonnull IJobPersistence theJobPersistence, } public JobStepExecutor newJobStepExecutor(@Nonnull JobInstance theInstance, WorkChunk theWorkChunk, @Nonnull JobWorkCursor theCursor) { - return new JobStepExecutor<>(myJobPersistence, myBatchJobSender, theInstance, theWorkChunk, theCursor, myJobStepExecutorSvc, myJobMaintenanceService, myJobDefinitionRegistry); + return new JobStepExecutor<>(myJobPersistence, theInstance, theWorkChunk, theCursor, myJobStepExecutorSvc, myJobMaintenanceService, myJobDefinitionRegistry); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java index d812b278caef..7c348888219e 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepDataSink.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.coordinator; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,23 +17,24 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.coordinator; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.maintenance.JobChunkProgressAccumulator; -import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunkData; +import ca.uhn.fhir.batch2.progress.InstanceProgress; import ca.uhn.fhir.batch2.progress.JobInstanceProgressCalculator; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.JsonUtil; import ca.uhn.fhir.util.Logs; +import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import java.util.Date; -import java.util.Optional; public class ReductionStepDataSink extends BaseDataSink { @@ -56,21 +55,39 @@ public ReductionStepDataSink(String theInstanceId, @Override public void accept(WorkChunkData theData) { String instanceId = getInstanceId(); - Optional instanceOp = myJobPersistence.fetchInstance(instanceId); - if (instanceOp.isPresent()) { - JobInstance instance = instanceOp.get(); + OT data = theData.getData(); + String dataString = JsonUtil.serialize(data, false); + JobChunkProgressAccumulator progressAccumulator = new JobChunkProgressAccumulator(); + JobInstanceProgressCalculator myJobInstanceProgressCalculator = new JobInstanceProgressCalculator(myJobPersistence, progressAccumulator, myJobDefinitionRegistry); + + InstanceProgress progress = myJobInstanceProgressCalculator.calculateInstanceProgress(instanceId); + boolean changed = myJobPersistence.updateInstance(instanceId, instance -> { + Validate.validState( + StatusEnum.FINALIZE.equals(instance.getStatus()), + "Job %s must be in FINALIZE state. In %s", instanceId, instance.getStatus()); if (instance.getReport() != null) { // last in wins - so we won't throw ourLog.error("Report has already been set. Now it is being overwritten. Last in will win!"); } - JobChunkProgressAccumulator progressAccumulator = new JobChunkProgressAccumulator(); - JobInstanceProgressCalculator myJobInstanceProgressCalculator = new JobInstanceProgressCalculator(myJobPersistence, progressAccumulator, myJobDefinitionRegistry); - myJobInstanceProgressCalculator.calculateInstanceProgressAndPopulateInstance(instance); + /* + * For jobs without a reduction step at the end, the maintenance service marks the job instance + * as COMPLETE when all chunks are complete, and calculates the final counts and progress. + * However, for jobs with a reduction step at the end the maintenance service stops working + * on the job while the job is in FINALIZE state, and this sink is ultimately responsible + * for marking the instance as COMPLETE at the end of the reduction. + * + * So, make sure we update the stats and counts before marking as complete here. + * + * I could envision a better setup where the stuff that the maintenance service touches + * is moved into separate DB tables or transactions away from the stuff that the + * reducer touches. If the two could never collide we wouldn't need this duplication + * here. Until then though, this is safer. + */ + + progress.updateInstance(instance); - OT data = theData.getData(); - String dataString = JsonUtil.serialize(data, false); instance.setReport(dataString); instance.setStatus(StatusEnum.COMPLETED); instance.setEndTime(new Date()); @@ -80,15 +97,13 @@ public void accept(WorkChunkData theData) { .addArgument(() -> JsonUtil.serialize(instance)) .log("New instance state: {}"); - myJobPersistence.updateInstance(instance); - - ourLog.info("Finalized job instance {} with report length {} chars", instance.getInstanceId(), dataString.length()); + return true; + }); - } else { - String msg = "No instance found with Id " + instanceId; - ourLog.error(msg); + if (!changed) { + ourLog.error("No instance found with Id {} in FINALIZE state", instanceId); - throw new JobExecutionFailedException(Msg.code(2097) + msg); + throw new JobExecutionFailedException(Msg.code(2097) + ("No instance found with Id " + instanceId)); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImpl.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImpl.java index 519fb42f7d28..d7dc9bd31d14 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImpl.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImpl.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.coordinator; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.coordinator; import ca.uhn.fhir.batch2.api.ChunkExecutionDetails; import ca.uhn.fhir.batch2.api.IJobPersistence; @@ -31,6 +30,7 @@ import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.model.sched.HapiJob; import ca.uhn.fhir.jpa.model.sched.IHasScheduledJobs; @@ -72,8 +72,8 @@ public class ReductionStepExecutorServiceImpl implements IReductionStepExecutorS private final IHapiTransactionService myTransactionService; private final Semaphore myCurrentlyExecuting = new Semaphore(1); private final AtomicReference myCurrentlyFinalizingInstanceId = new AtomicReference<>(); - private Timer myHeartbeatTimer; private final JobDefinitionRegistry myJobDefinitionRegistry; + private Timer myHeartbeatTimer; /** @@ -90,15 +90,17 @@ public ReductionStepExecutorServiceImpl(IJobPersistence theJobPersistence, IHapi @EventListener(ContextRefreshedEvent.class) public void start() { - myHeartbeatTimer = new Timer("batch2-reducer-heartbeat"); - myHeartbeatTimer.schedule(new HeartbeatTimerTask(), DateUtils.MILLIS_PER_MINUTE, DateUtils.MILLIS_PER_MINUTE); + if (myHeartbeatTimer == null) { + myHeartbeatTimer = new Timer("batch2-reducer-heartbeat"); + myHeartbeatTimer.schedule(new HeartbeatTimerTask(), DateUtils.MILLIS_PER_MINUTE, DateUtils.MILLIS_PER_MINUTE); + } } private void runHeartbeat() { String currentlyFinalizingInstanceId = myCurrentlyFinalizingInstanceId.get(); if (currentlyFinalizingInstanceId != null) { ourLog.info("Running heartbeat for instance: {}", currentlyFinalizingInstanceId); - executeInTransactionWithSynchronization(()->{ + executeInTransactionWithSynchronization(() -> { myJobPersistence.updateInstanceUpdateTime(currentlyFinalizingInstanceId); return null; }); @@ -107,7 +109,10 @@ private void runHeartbeat() { @EventListener(ContextClosedEvent.class) public void shutdown() { - myHeartbeatTimer.cancel(); + if (myHeartbeatTimer != null) { + myHeartbeatTimer.cancel(); + myHeartbeatTimer = null; + } } @@ -115,7 +120,7 @@ public void shutdown() { public void triggerReductionStep(String theInstanceId, JobWorkCursor theJobWorkCursor) { myInstanceIdToJobWorkCursor.putIfAbsent(theInstanceId, theJobWorkCursor); if (myCurrentlyExecuting.availablePermits() > 0) { - myReducerExecutor.submit(() -> reducerPass()); + myReducerExecutor.submit(this::reducerPass); } } @@ -135,6 +140,8 @@ public void reducerPass() { myInstanceIdToJobWorkCursor.remove(instanceId); } + } catch (Exception e) { + ourLog.error("Failed to execute reducer pass", e); } finally { myCurrentlyFinalizingInstanceId.set(null); myCurrentlyExecuting.release(); @@ -147,39 +154,35 @@ ReductionS JobDefinitionStep step = theJobWorkCursor.getCurrentStep(); - JobInstance instance = executeInTransactionWithSynchronization(() -> { - JobInstance currentInstance = myJobPersistence.fetchInstance(theInstanceId).orElseThrow(() -> new InternalErrorException("Unknown currentInstance: " + theInstanceId)); - boolean shouldProceed = false; - switch (currentInstance.getStatus()) { - case IN_PROGRESS: - case ERRORED: - if (myJobPersistence.markInstanceAsStatus(currentInstance.getInstanceId(), StatusEnum.FINALIZE)) { - ourLog.info("Job instance {} has been set to FINALIZE state - Beginning reducer step", currentInstance.getInstanceId()); - shouldProceed = true; - } - break; - case FINALIZE: - case COMPLETED: - case FAILED: - case QUEUED: - case CANCELLED: - break; - } - - if (!shouldProceed) { - ourLog.warn( - "JobInstance[{}] should not be finalized at this time. In memory status is {}. Reduction step will not rerun!" - + " This could be a long running reduction job resulting in the processed msg not being acknowledge," - + " or the result of a failed process or server restarting.", - currentInstance.getInstanceId(), - currentInstance.getStatus().name() - ); - return null; - } + JobInstance instance = executeInTransactionWithSynchronization(() -> + myJobPersistence.fetchInstance(theInstanceId).orElseThrow(() -> new InternalErrorException("Unknown instance: " + theInstanceId))); + + boolean shouldProceed = false; + switch (instance.getStatus()) { + case IN_PROGRESS: + case ERRORED: + // this will take a write lock on the JobInstance, preventing duplicates. + if (myJobPersistence.markInstanceAsStatus(instance.getInstanceId(), StatusEnum.FINALIZE)) { + ourLog.info("Job instance {} has been set to FINALIZE state - Beginning reducer step", instance.getInstanceId()); + shouldProceed = true; + } + break; + case FINALIZE: + case COMPLETED: + case FAILED: + case QUEUED: + case CANCELLED: + break; + } - return currentInstance; - }); - if (instance == null) { + if (!shouldProceed) { + ourLog.warn( + "JobInstance[{}] should not be finalized at this time. In memory status is {}. Reduction step will not rerun!" + + " This could be a long running reduction job resulting in the processed msg not being acknowledge," + + " or the result of a failed process or server restarting.", + instance.getInstanceId(), + instance.getStatus() + ); return new ReductionStepChunkProcessingResponse(false); } @@ -194,9 +197,8 @@ ReductionS try { executeInTransactionWithSynchronization(() -> { try (Stream chunkIterator = myJobPersistence.fetchAllWorkChunksForStepStream(instance.getInstanceId(), step.getStepId())) { - chunkIterator.forEach((chunk) -> { - processChunk(chunk, instance, parameters, reductionStepWorker, response, theJobWorkCursor); - }); + chunkIterator.forEach(chunk -> + processChunk(chunk, instance, parameters, reductionStepWorker, response, theJobWorkCursor)); } return null; }); @@ -216,7 +218,7 @@ ReductionS // complete the steps without making a new work chunk myJobPersistence.markWorkChunksWithStatusAndWipeData(instance.getInstanceId(), response.getSuccessfulChunkIds(), - StatusEnum.COMPLETED, + WorkChunkStatusEnum.COMPLETED, null // error message - none ); } @@ -225,7 +227,7 @@ ReductionS // mark any failed chunks as failed for aborting myJobPersistence.markWorkChunksWithStatusAndWipeData(instance.getInstanceId(), response.getFailedChunksIds(), - StatusEnum.FAILED, + WorkChunkStatusEnum.FAILED, "JOB ABORTED"); } return null; @@ -312,7 +314,7 @@ void processChunk(WorkChunk theChunk, ourLog.error(msg, e); theResponseObject.setSuccessful(false); - myJobPersistence.markWorkChunkAsFailed(theChunk.getId(), msg); + myJobPersistence.onWorkChunkFailed(theChunk.getId(), msg); } } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java index 9dd208be65b5..4ad7205f230b 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/StepExecutor.java @@ -26,18 +26,15 @@ import ca.uhn.fhir.batch2.api.JobStepFailedException; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; -import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.i18n.Msg; -import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.util.Logs; import org.apache.commons.lang3.Validate; import org.slf4j.Logger; -import java.util.Optional; - -import static ca.uhn.fhir.batch2.coordinator.WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT; - public class StepExecutor { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); private final IJobPersistence myJobPersistence; @@ -69,28 +66,16 @@ boolean ex chunkId, e); if (theStepExecutionDetails.hasAssociatedWorkChunk()) { - myJobPersistence.markWorkChunkAsFailed(chunkId, e.toString()); + myJobPersistence.onWorkChunkFailed(chunkId, e.toString()); } return false; } catch (Exception e) { if (theStepExecutionDetails.hasAssociatedWorkChunk()) { ourLog.error("Failure executing job {} step {}, marking chunk {} as ERRORED", jobDefinitionId, targetStepId, chunkId, e); - MarkWorkChunkAsErrorRequest parameters = new MarkWorkChunkAsErrorRequest(); - parameters.setChunkId(chunkId); - parameters.setErrorMsg(e.getMessage()); - Optional updatedOp = myJobPersistence.markWorkChunkAsErroredAndIncrementErrorCount(parameters); - if (updatedOp.isPresent()) { - WorkChunk chunk = updatedOp.get(); - - // see comments on MAX_CHUNK_ERROR_COUNT - if (chunk.getErrorCount() > MAX_CHUNK_ERROR_COUNT) { - String errorMsg = "Too many errors: " - + chunk.getErrorCount() - + ". Last error msg was " - + e.getMessage(); - myJobPersistence.markWorkChunkAsFailed(chunkId, errorMsg); - return false; - } + WorkChunkErrorEvent parameters = new WorkChunkErrorEvent(chunkId, e.getMessage()); + WorkChunkStatusEnum newStatus = myJobPersistence.onWorkChunkError(parameters); + if (newStatus == WorkChunkStatusEnum.FAILED) { + return false; } } else { ourLog.error("Failure executing job {} step {}, no associated work chunk", jobDefinitionId, targetStepId, e); @@ -99,7 +84,7 @@ boolean ex } catch (Throwable t) { ourLog.error("Unexpected failure executing job {} step {}", jobDefinitionId, targetStepId, t); if (theStepExecutionDetails.hasAssociatedWorkChunk()) { - myJobPersistence.markWorkChunkAsFailed(chunkId, t.toString()); + myJobPersistence.onWorkChunkFailed(chunkId, t.toString()); } return false; } @@ -108,10 +93,8 @@ boolean ex int recordsProcessed = outcome.getRecordsProcessed(); int recoveredErrorCount = theDataSink.getRecoveredErrorCount(); - myJobPersistence.markWorkChunkAsCompletedAndClearData(theStepExecutionDetails.getInstance().getInstanceId(), chunkId, recordsProcessed); - if (recoveredErrorCount > 0) { - myJobPersistence.incrementWorkChunkErrorCount(chunkId, recoveredErrorCount); - } + WorkChunkCompletionEvent event = new WorkChunkCompletionEvent(chunkId, recordsProcessed, recoveredErrorCount); + myJobPersistence.onWorkChunkCompletion(event); } return true; diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java deleted file mode 100644 index 885826b92d54..000000000000 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/SynchronizedJobPersistenceWrapper.java +++ /dev/null @@ -1,201 +0,0 @@ -package ca.uhn.fhir.batch2.coordinator; - -/*- - * #%L - * HAPI FHIR JPA Server - Batch2 Task Processor - * %% - * Copyright (C) 2014 - 2023 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -import ca.uhn.fhir.batch2.api.IJobPersistence; -import ca.uhn.fhir.batch2.api.JobOperationResultJson; -import ca.uhn.fhir.batch2.model.FetchJobInstancesRequest; -import ca.uhn.fhir.batch2.model.JobInstance; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; -import ca.uhn.fhir.batch2.model.StatusEnum; -import ca.uhn.fhir.batch2.model.WorkChunk; -import ca.uhn.fhir.batch2.models.JobInstanceFetchRequest; -import org.springframework.data.domain.Page; -import org.springframework.data.domain.Pageable; - -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Stream; - -public class SynchronizedJobPersistenceWrapper implements IJobPersistence { - - private final IJobPersistence myWrap; - - /** - * Constructor - */ - public SynchronizedJobPersistenceWrapper(IJobPersistence theJobPersistence) { - myWrap = theJobPersistence; - } - - @Override - public synchronized String storeWorkChunk(BatchWorkChunk theBatchWorkChunk) { - return myWrap.storeWorkChunk(theBatchWorkChunk); - } - - @Override - public synchronized Optional fetchWorkChunkSetStartTimeAndMarkInProgress(String theChunkId) { - return myWrap.fetchWorkChunkSetStartTimeAndMarkInProgress(theChunkId); - } - - @Override - public synchronized String storeNewInstance(JobInstance theInstance) { - return myWrap.storeNewInstance(theInstance); - } - - @Override - public synchronized Optional fetchInstance(String theInstanceId) { - return myWrap.fetchInstance(theInstanceId); - } - - @Override - public List fetchInstances(String theJobDefinitionId, Set theStatuses, Date theCutoff, Pageable thePageable) { - return myWrap.fetchInstances(theJobDefinitionId, theStatuses, theCutoff, thePageable); - } - - @Override - public synchronized List fetchInstances(FetchJobInstancesRequest theRequest, int theStart, int theBatchSize) { - return myWrap.fetchInstances(theRequest, theStart, theBatchSize); - } - - @Override - public synchronized List fetchInstances(int thePageSize, int thePageIndex) { - return myWrap.fetchInstances(thePageSize, thePageIndex); - } - - @Override - public List fetchRecentInstances(int thePageSize, int thePageIndex) { - return myWrap.fetchRecentInstances(thePageSize, thePageIndex); - } - - @Override - public List fetchInstancesByJobDefinitionIdAndStatus(String theJobDefinitionId, Set theRequestedStatuses, int thePageSize, int thePageIndex) { - return myWrap.fetchInstancesByJobDefinitionIdAndStatus(theJobDefinitionId, theRequestedStatuses, thePageSize, thePageIndex); - } - - @Override - public List fetchInstancesByJobDefinitionId(String theJobDefinitionId, int theCount, int theStart) { - return myWrap.fetchInstancesByJobDefinitionId(theJobDefinitionId, theCount, theStart); - } - - @Override - public Page fetchJobInstances(JobInstanceFetchRequest theRequest) { - return myWrap.fetchJobInstances(theRequest); - } - - @Override - public synchronized void markWorkChunkAsErroredAndIncrementErrorCount(String theChunkId, String theErrorMessage) { - myWrap.markWorkChunkAsErroredAndIncrementErrorCount(theChunkId, theErrorMessage); - } - - @Override - public Optional markWorkChunkAsErroredAndIncrementErrorCount(MarkWorkChunkAsErrorRequest theParameters) { - return myWrap.markWorkChunkAsErroredAndIncrementErrorCount(theParameters); - } - - @Override - public synchronized void markWorkChunkAsFailed(String theChunkId, String theErrorMessage) { - myWrap.markWorkChunkAsFailed(theChunkId, theErrorMessage); - } - - @Override - public synchronized void markWorkChunkAsCompletedAndClearData(String theInstanceId, String theChunkId, int theRecordsProcessed) { - myWrap.markWorkChunkAsCompletedAndClearData(theInstanceId, theChunkId, theRecordsProcessed); - } - - @Override - public void markWorkChunksWithStatusAndWipeData(String theInstanceId, List theChunkIds, StatusEnum theStatus, String theErrorMsg) { - myWrap.markWorkChunksWithStatusAndWipeData(theInstanceId, theChunkIds, theStatus, theErrorMsg); - } - - @Override - public void incrementWorkChunkErrorCount(String theChunkId, int theIncrementBy) { - myWrap.incrementWorkChunkErrorCount(theChunkId, theIncrementBy); - } - - @Override - public boolean canAdvanceInstanceToNextStep(String theInstanceId, String theCurrentStepId) { - return myWrap.canAdvanceInstanceToNextStep(theInstanceId, theCurrentStepId); - } - - @Override - public synchronized List fetchWorkChunksWithoutData(String theInstanceId, int thePageSize, int thePageIndex) { - return myWrap.fetchWorkChunksWithoutData(theInstanceId, thePageSize, thePageIndex); - } - - @Override - public Iterator fetchAllWorkChunksIterator(String theInstanceId, boolean theWithData) { - return myWrap.fetchAllWorkChunksIterator(theInstanceId, theWithData); - } - - @Override - public Iterator fetchAllWorkChunksForStepIterator(String theInstanceId, String theStepId) { - return myWrap.fetchAllWorkChunksForStepIterator(theInstanceId, theStepId); - } - - @Override - public Stream fetchAllWorkChunksForStepStream(String theInstanceId, String theStepId) { - return myWrap.fetchAllWorkChunksForStepStream(theInstanceId, theStepId); - } - - @Override - public synchronized boolean updateInstance(JobInstance theInstance) { - return myWrap.updateInstance(theInstance); - } - - @Override - public synchronized void deleteInstanceAndChunks(String theInstanceId) { - myWrap.deleteInstanceAndChunks(theInstanceId); - } - - @Override - public synchronized void deleteChunksAndMarkInstanceAsChunksPurged(String theInstanceId) { - myWrap.deleteChunksAndMarkInstanceAsChunksPurged(theInstanceId); - } - - @Override - public synchronized boolean markInstanceAsCompleted(String theInstanceId) { - return myWrap.markInstanceAsCompleted(theInstanceId); - } - - @Override - public boolean markInstanceAsStatus(String theInstance, StatusEnum theStatusEnum) { - return myWrap.markInstanceAsStatus(theInstance, theStatusEnum); - } - - @Override - public JobOperationResultJson cancelInstance(String theInstanceId) { - return myWrap.cancelInstance(theInstanceId); - } - - @Override - public List fetchallchunkidsforstepWithStatus(String theInstanceId, String theStepId, StatusEnum theStatusEnum) { - return myWrap.fetchallchunkidsforstepWithStatus(theInstanceId, theStepId, theStatusEnum); - } - - @Override - public synchronized void updateInstanceUpdateTime(String theInstanceId) { - myWrap.updateInstanceUpdateTime(theInstanceId); - } -} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java index d10cf18cd5d4..57ffcc3468a3 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChannelMessageHandler.java @@ -20,7 +20,6 @@ * #L% */ - import ca.uhn.fhir.batch2.api.IJobMaintenanceService; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.channel.BatchJobSender; @@ -29,12 +28,9 @@ import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.JobWorkNotificationJsonMessage; -import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; -import ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater; -import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; +import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.util.Logs; -import org.apache.commons.lang3.Validate; import org.slf4j.Logger; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHandler; @@ -42,6 +38,7 @@ import javax.annotation.Nonnull; import java.util.Optional; +import java.util.function.Supplier; /** * This handler receives batch work request messages and performs the batch work requested by the message @@ -51,17 +48,18 @@ class WorkChannelMessageHandler implements MessageHandler { private final IJobPersistence myJobPersistence; private final JobDefinitionRegistry myJobDefinitionRegistry; private final JobStepExecutorFactory myJobStepExecutorFactory; - private final JobInstanceStatusUpdater myJobInstanceStatusUpdater; + private final IHapiTransactionService myHapiTransactionService; WorkChannelMessageHandler(@Nonnull IJobPersistence theJobPersistence, @Nonnull JobDefinitionRegistry theJobDefinitionRegistry, @Nonnull BatchJobSender theBatchJobSender, @Nonnull WorkChunkProcessor theExecutorSvc, - @Nonnull IJobMaintenanceService theJobMaintenanceService) { + @Nonnull IJobMaintenanceService theJobMaintenanceService, + IHapiTransactionService theHapiTransactionService) { myJobPersistence = theJobPersistence; myJobDefinitionRegistry = theJobDefinitionRegistry; + myHapiTransactionService = theHapiTransactionService; myJobStepExecutorFactory = new JobStepExecutorFactory(theJobPersistence, theBatchJobSender, theExecutorSvc, theJobMaintenanceService, theJobDefinitionRegistry); - myJobInstanceStatusUpdater = new JobInstanceStatusUpdater(theJobPersistence, theJobDefinitionRegistry); } @Override @@ -69,55 +67,185 @@ public void handleMessage(@Nonnull Message theMessage) throws MessagingExcept handleWorkChannelMessage((JobWorkNotificationJsonMessage) theMessage); } - private void handleWorkChannelMessage(JobWorkNotificationJsonMessage theMessage) { - JobWorkNotification workNotification = theMessage.getPayload(); - ourLog.info("Received work notification for {}", workNotification); + /** + * Workflow scratchpad for processing a single chunk message. + */ + class MessageProcess { + final JobWorkNotification myWorkNotification; + String myChunkId; + WorkChunk myWorkChunk; + JobWorkCursor myCursor; + JobInstance myJobInstance; + JobDefinition myJobDefinition; + JobStepExecutor myStepExector; + + MessageProcess(JobWorkNotification theWorkNotification) { + myWorkNotification = theWorkNotification; + } + + /** + * Save the chunkId and validate. + */ + Optional validateChunkId() { + myChunkId = myWorkNotification.getChunkId(); + if (myChunkId == null) { + ourLog.error("Received work notification with null chunkId: {}", myWorkNotification); + return Optional.empty(); + } + return Optional.of(this); + } + + Optional loadJobDefinitionOrThrow() { + String jobDefinitionId = myWorkNotification.getJobDefinitionId(); + int jobDefinitionVersion = myWorkNotification.getJobDefinitionVersion(); + + // Do not catch this exception - that will discard this chunk. + // Failing to load a job definition probably means this is an old process during upgrade. + // Retry those until this node is killed/restarted. + myJobDefinition = myJobDefinitionRegistry.getJobDefinitionOrThrowException(jobDefinitionId, jobDefinitionVersion); + return Optional.of(this); + } - String chunkId = workNotification.getChunkId(); - Validate.notNull(chunkId); + /** + * Fetch the job instance including the job definition. + */ + Optional loadJobInstance() { + return myJobPersistence.fetchInstance(myWorkNotification.getInstanceId()) + .or(()->{ + ourLog.error("No instance {} exists for chunk notification {}", myWorkNotification.getInstanceId(), myWorkNotification); + return Optional.empty(); + }) + .map(instance->{ + myJobInstance = instance; + instance.setJobDefinition(myJobDefinition); + return this; + }); + } + + /** + * Load the chunk, and mark it as dequeued. + */ + Optional updateChunkStatusAndValidate() { + return myJobPersistence.onWorkChunkDequeue(myChunkId) + .or(()->{ + ourLog.error("Unable to find chunk with ID {} - Aborting. {}", myChunkId, myWorkNotification); + return Optional.empty(); + }) + .map(chunk->{ + myWorkChunk = chunk; + ourLog.debug("Worker picked up chunk. [chunkId={}, stepId={}, startTime={}]", myChunkId, myWorkChunk.getTargetStepId(), myWorkChunk.getStartTime()); + return this; + }); + } - JobWorkCursor cursor = null; - WorkChunk workChunk = null; - Optional chunkOpt = myJobPersistence.fetchWorkChunkSetStartTimeAndMarkInProgress(chunkId); - if (chunkOpt.isEmpty()) { - ourLog.error("Unable to find chunk with ID {} - Aborting", chunkId); - return; + /** + * Move QUEUED jobs to IN_PROGRESS, and make sure we are not already in final state. + */ + Optional updateAndValidateJobStatus() { + ourLog.trace("Check status {} of job {} for chunk {}", myJobInstance.getStatus(), myJobInstance.getInstanceId(), myChunkId); + switch (myJobInstance.getStatus()) { + case QUEUED: + // Update the job as started. + myJobPersistence.onChunkDequeued(myJobInstance.getInstanceId()); + break; + + case IN_PROGRESS: + case ERRORED: + case FINALIZE: + // normal processing + break; + + case COMPLETED: + // this is an error, but we can't do much about it. + ourLog.error("Received chunk {}, but job instance is {}. Skipping.", myChunkId, myJobInstance.getStatus()); + return Optional.empty(); + + case CANCELLED: + case FAILED: + default: + // should we mark the chunk complete/failed for any of these skipped? + ourLog.info("Skipping chunk {} because job instance is {}", myChunkId, myJobInstance.getStatus()); + return Optional.empty(); + } + + return Optional.of(this); } - workChunk = chunkOpt.get(); - ourLog.debug("Worker picked up chunk. [chunkId={}, stepId={}, startTime={}]", chunkId, workChunk.getTargetStepId(), workChunk.getStartTime()); - cursor = buildCursorFromNotification(workNotification); - Validate.isTrue(workChunk.getTargetStepId().equals(cursor.getCurrentStepId()), "Chunk %s has target step %s but expected %s", chunkId, workChunk.getTargetStepId(), cursor.getCurrentStepId()); + Optional buildCursor() { - Optional instanceOpt = myJobPersistence.fetchInstance(workNotification.getInstanceId()); - JobInstance instance = instanceOpt.orElseThrow(() -> new InternalErrorException("Unknown instance: " + workNotification.getInstanceId())); - markInProgressIfQueued(instance); - myJobDefinitionRegistry.setJobDefinition(instance); - String instanceId = instance.getInstanceId(); + myCursor = JobWorkCursor.fromJobDefinitionAndRequestedStepId(myJobDefinition, myWorkNotification.getTargetStepId()); - if (instance.isCancelled()) { - ourLog.info("Skipping chunk {} because job instance is cancelled", chunkId); - myJobPersistence.markInstanceAsStatus(instanceId, StatusEnum.CANCELLED); - return; + if (!myWorkChunk.getTargetStepId().equals(myCursor.getCurrentStepId())) { + ourLog.error("Chunk {} has target step {} but expected {}", myChunkId, myWorkChunk.getTargetStepId(), myCursor.getCurrentStepId()); + return Optional.empty(); + } + return Optional.of(this); } - JobStepExecutor stepExecutor = myJobStepExecutorFactory.newJobStepExecutor(instance, workChunk, cursor); - stepExecutor.executeStep(); - } + public Optional buildStepExecutor() { + this.myStepExector = myJobStepExecutorFactory.newJobStepExecutor(this.myJobInstance, this.myWorkChunk, this.myCursor); - private void markInProgressIfQueued(JobInstance theInstance) { - if (theInstance.getStatus() == StatusEnum.QUEUED) { - myJobInstanceStatusUpdater.updateInstanceStatus(theInstance, StatusEnum.IN_PROGRESS); + return Optional.of(this); } } - private JobWorkCursor buildCursorFromNotification(JobWorkNotification workNotification) { - String jobDefinitionId = workNotification.getJobDefinitionId(); - int jobDefinitionVersion = workNotification.getJobDefinitionVersion(); + private void handleWorkChannelMessage(JobWorkNotificationJsonMessage theMessage) { + JobWorkNotification workNotification = theMessage.getPayload(); + ourLog.info("Received work notification for {}", workNotification); + + // There are three paths through this code: + // 1. Normal execution. We validate, load, update statuses, all in a tx. Then we proces the chunk. + // 2. Discard chunk. If some validation fails (e.g. no chunk with that id), we log and discard the chunk. + // Probably a db rollback, with a stale queue. + // 3. Fail and retry. If we throw an exception out of here, Spring will put the queue message back, and redeliver later. + // + // We use Optional chaining here to simplify all the cases where we short-circuit exit. + // A step that returns an empty Optional means discard the chunk. + // + executeInTxRollbackWhenEmpty(() -> ( + // Use a chain of Optional flatMap to handle all the setup short-circuit exits cleanly. + Optional.of(new MessageProcess(workNotification)) + // validate and load info + .flatMap(MessageProcess::validateChunkId) + // no job definition should be retried - we must be a stale process encountering a new job definition. + .flatMap(MessageProcess::loadJobDefinitionOrThrow) + .flatMap(MessageProcess::loadJobInstance) + // update statuses now in the db: QUEUED->IN_PROGRESS + .flatMap(MessageProcess::updateChunkStatusAndValidate) + .flatMap(MessageProcess::updateAndValidateJobStatus) + // ready to execute + .flatMap(MessageProcess::buildCursor) + .flatMap(MessageProcess::buildStepExecutor) + )) + .ifPresentOrElse( + // all the setup is happy and committed. Do the work. + process -> process.myStepExector.executeStep(), + // discard the chunk + () -> ourLog.debug("Discarding chunk notification {}", workNotification) + ); - JobDefinition definition = myJobDefinitionRegistry.getJobDefinitionOrThrowException(jobDefinitionId, jobDefinitionVersion); + } - return JobWorkCursor.fromJobDefinitionAndRequestedStepId(definition, workNotification.getTargetStepId()); + /** + * Run theCallback in TX, rolling back if the supplied Optional is empty. + */ + Optional executeInTxRollbackWhenEmpty(Supplier> theCallback) { + return myHapiTransactionService.withRequest(null) + .execute(theTransactionStatus -> { + + // run the processing + Optional setupProcessing = theCallback.get(); + + if (setupProcessing.isEmpty()) { + // If any setup failed, roll back the chunk and instance status changes. + ourLog.debug("WorkChunk setup tx rollback"); + theTransactionStatus.setRollbackOnly(); + } + // else COMMIT the work. + + return setupProcessing; + }); } + } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java index 06014236cc17..fb05234939f7 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessor.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.coordinator; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,6 +17,7 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.coordinator; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.IJobStepWorker; @@ -30,7 +29,6 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.WorkChunk; -import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.Logs; import org.apache.commons.lang3.Validate; @@ -58,9 +56,7 @@ public class WorkChunkProcessor { private final BatchJobSender myBatchJobSender; private final StepExecutor myStepExecutor; - public WorkChunkProcessor(IJobPersistence theJobPersistence, - BatchJobSender theSender, - IHapiTransactionService theTransactionService) { + public WorkChunkProcessor(IJobPersistence theJobPersistence, BatchJobSender theSender) { myJobPersistence = theJobPersistence; myBatchJobSender = theSender; myStepExecutor = new StepExecutor(theJobPersistence); diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobChunkProgressAccumulator.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobChunkProgressAccumulator.java index f1671f3ad375..b6cbe3c2d267 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobChunkProgressAccumulator.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobChunkProgressAccumulator.java @@ -21,8 +21,8 @@ */ -import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.util.Logs; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; @@ -38,11 +38,10 @@ import static java.util.Collections.emptyList; import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; -import static org.slf4j.LoggerFactory.getLogger; /** - * While performing cleanup, the cleanup job loads all of the known - * work chunks to examine their status. This bean collects the counts that + * While performing cleanup, the cleanup job loads all work chunks + * to examine their status. This bean collects the counts that * are found, so that they can be reused for maintenance jobs without * needing to hit the database a second time. */ @@ -52,15 +51,11 @@ public class JobChunkProgressAccumulator { private final Set myConsumedInstanceAndChunkIds = new HashSet<>(); private final Multimap myInstanceIdToChunkStatuses = ArrayListMultimap.create(); - int countChunksWithStatus(String theInstanceId, String theStepId, StatusEnum... theStatuses) { - return getChunkIdsWithStatus(theInstanceId, theStepId, theStatuses).size(); - } - int getTotalChunkCountForInstanceAndStep(String theInstanceId, String theStepId) { return myInstanceIdToChunkStatuses.get(theInstanceId).stream().filter(chunkCount -> chunkCount.myStepId.equals(theStepId)).collect(Collectors.toList()).size(); } - public List getChunkIdsWithStatus(String theInstanceId, String theStepId, StatusEnum... theStatuses) { + public List getChunkIdsWithStatus(String theInstanceId, String theStepId, WorkChunkStatusEnum... theStatuses) { return getChunkStatuses(theInstanceId).stream() .filter(t -> t.myStepId.equals(theStepId)) .filter(t -> ArrayUtils.contains(theStatuses, t.myStatus)) @@ -81,7 +76,7 @@ public void addChunk(WorkChunk theChunk) { // Note: If chunks are being written while we're executing, we may see the same chunk twice. This // check avoids adding it twice. if (myConsumedInstanceAndChunkIds.add(instanceId + " " + chunkId)) { - ourLog.debug("Adding chunk to accumulator. [chunkId={}, instanceId={}, status={}]", chunkId, instanceId, theChunk.getStatus()); + ourLog.debug("Adding chunk to accumulator. [chunkId={}, instanceId={}, status={}, step={}]", chunkId, instanceId, theChunk.getStatus(), theChunk.getTargetStepId()); myInstanceIdToChunkStatuses.put(instanceId, new ChunkStatusCountValue(chunkId, theChunk.getTargetStepId(), theChunk.getStatus())); } } @@ -89,9 +84,9 @@ public void addChunk(WorkChunk theChunk) { private static class ChunkStatusCountValue { public final String myChunkId; public final String myStepId; - public final StatusEnum myStatus; + public final WorkChunkStatusEnum myStatus; - private ChunkStatusCountValue(String theChunkId, String theStepId, StatusEnum theStatus) { + private ChunkStatusCountValue(String theChunkId, String theStepId, WorkChunkStatusEnum theStatus) { myChunkId = theChunkId; myStepId = theStepId; myStatus = theStatus; diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java index 0849bfe85ab0..bb9f421be153 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobInstanceProcessor.java @@ -29,6 +29,8 @@ import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; +import ca.uhn.fhir.batch2.progress.InstanceProgress; import ca.uhn.fhir.batch2.progress.JobInstanceProgressCalculator; import ca.uhn.fhir.batch2.progress.JobInstanceStatusUpdater; import ca.uhn.fhir.model.api.IModelJson; @@ -52,7 +54,7 @@ public class JobInstanceProcessor { private final String myInstanceId; private final JobDefinitionRegistry myJobDefinitionegistry; - JobInstanceProcessor(IJobPersistence theJobPersistence, + public JobInstanceProcessor(IJobPersistence theJobPersistence, BatchJobSender theBatchJobSender, String theInstanceId, JobChunkProgressAccumulator theProgressAccumulator, @@ -65,7 +67,7 @@ public class JobInstanceProcessor { myReductionStepExecutorService = theReductionStepExecutorService; myJobDefinitionegistry = theJobDefinitionRegistry; myJobInstanceProgressCalculator = new JobInstanceProgressCalculator(theJobPersistence, theProgressAccumulator, theJobDefinitionRegistry); - myJobInstanceStatusUpdater = new JobInstanceStatusUpdater(theJobPersistence, theJobDefinitionRegistry); + myJobInstanceStatusUpdater = new JobInstanceStatusUpdater(theJobDefinitionRegistry); } public void process() { @@ -76,19 +78,31 @@ public void process() { if (theInstance == null) { return; } - - handleCancellation(theInstance); + + boolean cancelUpdate = handleCancellation(theInstance); + if (cancelUpdate) { + // reload after update + theInstance = myJobPersistence.fetchInstance(myInstanceId).orElseThrow(); + } cleanupInstance(theInstance); triggerGatedExecutions(theInstance); ourLog.debug("Finished job processing: {} - {}", myInstanceId, stopWatch); } - private void handleCancellation(JobInstance theInstance) { + private boolean handleCancellation(JobInstance theInstance) { if (theInstance.isPendingCancellationRequest()) { - theInstance.setErrorMessage(buildCancelledMessage(theInstance)); - myJobInstanceStatusUpdater.setCancelled(theInstance); + String errorMessage = buildCancelledMessage(theInstance); + ourLog.info("Job {} moving to CANCELLED", theInstance.getInstanceId()); + return myJobPersistence.updateInstance(theInstance.getInstanceId(), instance -> { + boolean changed = myJobInstanceStatusUpdater.updateInstanceStatus(instance, StatusEnum.CANCELLED); + if (changed) { + instance.setErrorMessage(errorMessage); + } + return changed; + }); } + return false; } private String buildCancelledMessage(JobInstance theInstance) { @@ -105,12 +119,12 @@ private void cleanupInstance(JobInstance theInstance) { // If we're still QUEUED, there are no stats to calculate break; case FINALIZE: - // If we're in FINALIZE, the reduction step is working so we should stay out of the way until it + // If we're in FINALIZE, the reduction step is working, so we should stay out of the way until it // marks the job as COMPLETED return; case IN_PROGRESS: case ERRORED: - myJobInstanceProgressCalculator.calculateAndStoreInstanceProgress(theInstance); + myJobInstanceProgressCalculator.calculateAndStoreInstanceProgress(theInstance.getInstanceId()); break; case COMPLETED: case FAILED: @@ -124,11 +138,15 @@ private void cleanupInstance(JobInstance theInstance) { } if (theInstance.isFinished() && !theInstance.isWorkChunksPurged()) { - myJobInstanceProgressCalculator.calculateInstanceProgressAndPopulateInstance(theInstance); - - theInstance.setWorkChunksPurged(true); myJobPersistence.deleteChunksAndMarkInstanceAsChunksPurged(theInstance.getInstanceId()); - myJobPersistence.updateInstance(theInstance); + + InstanceProgress progress = myJobInstanceProgressCalculator.calculateInstanceProgress(theInstance.getInstanceId()); + + myJobPersistence.updateInstance(theInstance.getInstanceId(), instance->{ + progress.updateInstance(instance); + instance.setWorkChunksPurged(true); + return true; + }); } } @@ -147,7 +165,7 @@ private boolean purgeExpiredInstance(JobInstance theInstance) { private void triggerGatedExecutions(JobInstance theInstance) { if (!theInstance.isRunning()) { ourLog.debug("JobInstance {} is not in a \"running\" state. Status {}", - theInstance.getInstanceId(), theInstance.getStatus().name()); + theInstance.getInstanceId(), theInstance.getStatus()); return; } @@ -186,19 +204,37 @@ private void triggerGatedExecutions(JobInstance theInstance) { private void processChunksForNextSteps(JobInstance theInstance, String nextStepId) { String instanceId = theInstance.getInstanceId(); - List queuedChunksForNextStep = myProgressAccumulator.getChunkIdsWithStatus(instanceId, nextStepId, StatusEnum.QUEUED); + List queuedChunksForNextStep = myProgressAccumulator.getChunkIdsWithStatus(instanceId, nextStepId, WorkChunkStatusEnum.QUEUED); int totalChunksForNextStep = myProgressAccumulator.getTotalChunkCountForInstanceAndStep(instanceId, nextStepId); if (totalChunksForNextStep != queuedChunksForNextStep.size()) { ourLog.debug("Total ProgressAccumulator QUEUED chunk count does not match QUEUED chunk size! [instanceId={}, stepId={}, totalChunks={}, queuedChunks={}]", instanceId, nextStepId, totalChunksForNextStep, queuedChunksForNextStep.size()); } - List chunksToSubmit = myJobPersistence.fetchallchunkidsforstepWithStatus(instanceId, nextStepId, StatusEnum.QUEUED); + // Note on sequence: we don't have XA transactions, and are talking to two stores (JPA + Queue) + // Sequence: 1 - So we run the query to minimize the work overlapping. + List chunksToSubmit = myJobPersistence.fetchAllChunkIdsForStepWithStatus(instanceId, nextStepId, WorkChunkStatusEnum.QUEUED); + // Sequence: 2 - update the job step so the workers will process them. + boolean changed = myJobPersistence.updateInstance(instanceId, instance -> { + if (instance.getCurrentGatedStepId().equals(nextStepId)) { + // someone else beat us here. No changes + return false; + } + instance.setCurrentGatedStepId(nextStepId); + return true; + }); + if (!changed) { + // we collided with another maintenance job. + return; + } + + // DESIGN GAP: if we die here, these chunks will never be queued. + // Need a WAITING stage before QUEUED for chunks, so we can catch them. + + // Sequence: 3 - send the notifications for (String nextChunkId : chunksToSubmit) { JobWorkNotification workNotification = new JobWorkNotification(theInstance, nextStepId, nextChunkId); myBatchJobSender.sendWorkChannelMessage(workNotification); } ourLog.debug("Submitted a batch of chunks for processing. [chunkCount={}, instanceId={}, stepId={}]", chunksToSubmit.size(), instanceId, nextStepId); - theInstance.setCurrentGatedStepId(nextStepId); - myJobPersistence.updateInstance(theInstance); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImpl.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImpl.java index 72224e13e113..9f55d08be077 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImpl.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImpl.java @@ -201,6 +201,7 @@ public void runMaintenancePass() { return; } try { + ourLog.info("Maintenance pass starting."); doMaintenancePass(); } finally { myRunMaintenanceSemaphore.release(); @@ -220,7 +221,7 @@ private void doMaintenancePass() { myJobDefinitionRegistry.setJobDefinition(instance); JobInstanceProcessor jobInstanceProcessor = new JobInstanceProcessor(myJobPersistence, myBatchJobSender, instanceId, progressAccumulator, myReductionStepExecutorService, myJobDefinitionRegistry); - ourLog.debug("Triggering maintenance process for instance {} in status {}", instanceId, instance.getStatus().name()); + ourLog.debug("Triggering maintenance process for instance {} in status {}", instanceId, instance.getStatus()); jobInstanceProcessor.process(); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/BaseWorkChunkEvent.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/BaseWorkChunkEvent.java new file mode 100644 index 000000000000..245c709ab0ed --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/BaseWorkChunkEvent.java @@ -0,0 +1,58 @@ +package ca.uhn.fhir.batch2.model; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Payloads for WorkChunk state transitions. + * Some events in the state-machine update the chunk metadata (e.g. error message). + * This class provides a base-class for those event objects. + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ +public abstract class BaseWorkChunkEvent { + protected final String myChunkId; + + protected BaseWorkChunkEvent(String theChunkId) { + myChunkId = theChunkId; + } + + public String getChunkId() { + return myChunkId; + } + + @Override + public boolean equals(Object theO) { + if (this == theO) return true; + + if (theO == null || getClass() != theO.getClass()) return false; + + BaseWorkChunkEvent that = (BaseWorkChunkEvent) theO; + + return new EqualsBuilder().append(myChunkId, that.myChunkId).isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).append(myChunkId).toHashCode(); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobInstance.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobInstance.java index 7df257312926..c61cda3d7c92 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobInstance.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/JobInstance.java @@ -24,8 +24,8 @@ import ca.uhn.fhir.jpa.util.JsonDateDeserializer; import ca.uhn.fhir.jpa.util.JsonDateSerializer; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.util.JsonUtil; import ca.uhn.fhir.util.Logs; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; @@ -36,7 +36,13 @@ import static org.apache.commons.lang3.StringUtils.isBlank; -public class JobInstance extends JobInstanceStartRequest implements IModelJson, IJobInstance { +public class JobInstance implements IModelJson, IJobInstance { + + @JsonProperty(value = "jobDefinitionId") + private String myJobDefinitionId; + + @JsonProperty(value = "parameters") + private String myParameters; @JsonProperty(value = "jobDefinitionVersion") private int myJobDefinitionVersion; @@ -115,7 +121,8 @@ public JobInstance() { * Copy constructor */ public JobInstance(JobInstance theJobInstance) { - super(theJobInstance); + setJobDefinitionId(theJobInstance.getJobDefinitionId()); + setParameters(theJobInstance.getParameters()); setCancelled(theJobInstance.isCancelled()); setFastTracking(theJobInstance.isFastTracking()); setCombinedRecordsProcessed(theJobInstance.getCombinedRecordsProcessed()); @@ -137,6 +144,34 @@ public JobInstance(JobInstance theJobInstance) { setReport(theJobInstance.getReport()); } + + public String getJobDefinitionId() { + return myJobDefinitionId; + } + + public void setJobDefinitionId(String theJobDefinitionId) { + myJobDefinitionId = theJobDefinitionId; + } + + public String getParameters() { + return myParameters; + } + + public void setParameters(String theParameters) { + myParameters = theParameters; + } + + public T getParameters(Class theType) { + if (myParameters == null) { + return null; + } + return JsonUtil.deserialize(myParameters, theType); + } + + public void setParameters(IModelJson theParameters) { + myParameters = JsonUtil.serializeOrInvalidRequest(theParameters); + } + public void setUpdateTime(Date theUpdateTime) { myUpdateTime = theUpdateTime; } @@ -332,6 +367,7 @@ public String toString() { .append("jobDefinitionId", getJobDefinitionId() + "/" + myJobDefinitionVersion) .append("instanceId", myInstanceId) .append("status", myStatus) + .append("myCancelled", myCancelled) .append("createTime", myCreateTime) .append("startTime", myStartTime) .append("endTime", myEndTime) @@ -369,7 +405,7 @@ public boolean isRunning() { case FAILED: case CANCELLED: default: - Logs.getBatchTroubleshootingLog().debug("Status {} is considered \"not running\"", getStatus().name()); + Logs.getBatchTroubleshootingLog().debug("Status {} is considered \"not running\"", myStatus); } return false; } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/MarkWorkChunkAsErrorRequest.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/MarkWorkChunkAsErrorRequest.java deleted file mode 100644 index 9b787e1c226f..000000000000 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/MarkWorkChunkAsErrorRequest.java +++ /dev/null @@ -1,56 +0,0 @@ -package ca.uhn.fhir.batch2.model; - -/*- - * #%L - * HAPI FHIR JPA Server - Batch2 Task Processor - * %% - * Copyright (C) 2014 - 2023 Smile CDR, Inc. - * %% - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * #L% - */ - -public class MarkWorkChunkAsErrorRequest { - private String myChunkId; - - private String myErrorMsg; - - private boolean myIncludeData; - - public String getChunkId() { - return myChunkId; - } - - public MarkWorkChunkAsErrorRequest setChunkId(String theChunkId) { - myChunkId = theChunkId; - return this; - } - - public String getErrorMsg() { - return myErrorMsg; - } - - public MarkWorkChunkAsErrorRequest setErrorMsg(String theErrorMsg) { - myErrorMsg = theErrorMsg; - return this; - } - - public boolean isIncludeData() { - return myIncludeData; - } - - public MarkWorkChunkAsErrorRequest setIncludeData(boolean theIncludeData) { - myIncludeData = theIncludeData; - return this; - } -} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/StatusEnum.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/StatusEnum.java index 759da90f02bb..bb2b71a4719e 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/StatusEnum.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/StatusEnum.java @@ -22,13 +22,19 @@ import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.util.Logs; +import com.google.common.collect.Maps; import org.slf4j.Logger; import javax.annotation.Nonnull; import java.util.Collections; +import java.util.EnumMap; import java.util.EnumSet; +import java.util.Map; import java.util.Set; +/** + * Status of a Batch2 Job Instance. + */ public enum StatusEnum { /** @@ -55,7 +61,7 @@ public enum StatusEnum { * Task execution resulted in an error but the error may be transient (or transient status is unknown). * Retrying may result in success. */ - ERRORED(true, true, false), + ERRORED(true, false, true), /** * Task has failed and is known to be unrecoverable. There is no reason to believe that retrying will @@ -70,6 +76,32 @@ public enum StatusEnum { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); + /** Map from state to Set of legal inbound states */ + static final Map> ourFromStates; + /** Map from state to Set of legal outbound states */ + static final Map> ourToStates; + + static { + EnumMap> fromStates = new EnumMap<>(StatusEnum.class); + EnumMap> toStates = new EnumMap<>(StatusEnum.class); + + for (StatusEnum nextEnum: StatusEnum.values()) { + fromStates.put(nextEnum, EnumSet.noneOf(StatusEnum.class)); + toStates.put(nextEnum, EnumSet.noneOf(StatusEnum.class)); + } + for (StatusEnum nextPriorEnum: StatusEnum.values()) { + for (StatusEnum nextNextEnum: StatusEnum.values()) { + if (isLegalStateTransition(nextPriorEnum, nextNextEnum)) { + fromStates.get(nextNextEnum).add(nextPriorEnum); + toStates.get(nextPriorEnum).add(nextNextEnum); + } + } + } + + ourFromStates = Maps.immutableEnumMap(fromStates); + ourToStates = Maps.immutableEnumMap(toStates); + } + private final boolean myIncomplete; private final boolean myEnded; private final boolean myIsCancellable; @@ -130,7 +162,6 @@ public static Set getNotEndedStatuses() { return retVal; } - @Nonnull private static void initializeStaticEndedStatuses() { EnumSet endedSet = EnumSet.noneOf(StatusEnum.class); EnumSet notEndedSet = EnumSet.noneOf(StatusEnum.class); @@ -146,10 +177,7 @@ private static void initializeStaticEndedStatuses() { } public static boolean isLegalStateTransition(StatusEnum theOrigStatus, StatusEnum theNewStatus) { - if (theOrigStatus == theNewStatus) { - return true; - } - Boolean canTransition; + boolean canTransition; switch (theOrigStatus) { case QUEUED: // initial state can transition to anything @@ -159,30 +187,29 @@ public static boolean isLegalStateTransition(StatusEnum theOrigStatus, StatusEnu canTransition = theNewStatus != QUEUED; break; case ERRORED: - canTransition = theNewStatus == FAILED || theNewStatus == COMPLETED || theNewStatus == CANCELLED; + canTransition = theNewStatus == FAILED || theNewStatus == COMPLETED || theNewStatus == CANCELLED || theNewStatus == ERRORED; break; - case COMPLETED: case CANCELLED: - case FAILED: // terminal state cannot transition canTransition = false; break; + case COMPLETED: + canTransition = false; + break; + case FAILED: + canTransition = theNewStatus == FAILED; + break; case FINALIZE: canTransition = theNewStatus != QUEUED && theNewStatus != IN_PROGRESS; break; default: - canTransition = null; - break; + throw new IllegalStateException(Msg.code(2131) + "Unknown batch state " + theOrigStatus); } - if (canTransition == null){ - throw new IllegalStateException(Msg.code(2131) + "Unknown batch state " + theOrigStatus); - } else { - if (!canTransition) { - ourLog.trace("Tried to execute an illegal state transition. [origStatus={}, newStatus={}]", theOrigStatus, theNewStatus); - } - return canTransition; + if (!canTransition) { + ourLog.trace("Tried to execute an illegal state transition. [origStatus={}, newStatus={}]", theOrigStatus, theNewStatus); } + return canTransition; } public boolean isIncomplete() { @@ -196,4 +223,18 @@ public boolean isEnded() { public boolean isCancellable() { return myIsCancellable; } + + /** + * States that may transition to this state. + */ + public Set getPriorStates() { + return ourFromStates.get(this); + } + + /** + * States this state may transotion to. + */ + public Set getNextStates() { + return ourToStates.get(this); + } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunk.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunk.java index 452d9ff30b34..40c0c7e28a00 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunk.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunk.java @@ -34,16 +34,23 @@ import static org.apache.commons.lang3.StringUtils.isNotBlank; +/** + * Payload for step processing. + * Implements a state machine on {@link WorkChunkStatusEnum}. + * + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ public class WorkChunk implements IModelJson { @JsonProperty("id") private String myId; @JsonProperty("sequence") + // TODO MB danger - these repeat with a job or even a single step. They start at 0 for every parent chunk. Review after merge. private int mySequence; @JsonProperty("status") - private StatusEnum myStatus; + private WorkChunkStatusEnum myStatus; @JsonProperty("jobDefinitionId") private String myJobDefinitionId; @@ -131,11 +138,11 @@ public WorkChunk setRecordsProcessed(Integer theRecordsProcessed) { return this; } - public StatusEnum getStatus() { + public WorkChunkStatusEnum getStatus() { return myStatus; } - public WorkChunk setStatus(StatusEnum theStatus) { + public WorkChunk setStatus(WorkChunkStatusEnum theStatus) { myStatus = theStatus; return this; } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCompletionEvent.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCompletionEvent.java new file mode 100644 index 000000000000..34d355ffaddc --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCompletionEvent.java @@ -0,0 +1,62 @@ +package ca.uhn.fhir.batch2.model; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Payload for the work-chunk completion event with the record and error counts. + */ +public class WorkChunkCompletionEvent extends BaseWorkChunkEvent { + int myRecordsProcessed; + int myRecoveredErrorCount; + + public WorkChunkCompletionEvent(String theChunkId, int theRecordsProcessed, int theRecoveredErrorCount) { + super(theChunkId); + myRecordsProcessed = theRecordsProcessed; + myRecoveredErrorCount = theRecoveredErrorCount; + } + + public int getRecordsProcessed() { + return myRecordsProcessed; + } + + public int getRecoveredErrorCount() { + return myRecoveredErrorCount; + } + + @Override + public boolean equals(Object theO) { + if (this == theO) return true; + + if (theO == null || getClass() != theO.getClass()) return false; + + WorkChunkCompletionEvent that = (WorkChunkCompletionEvent) theO; + + return new EqualsBuilder().appendSuper(super.equals(theO)).append(myRecordsProcessed, that.myRecordsProcessed).append(myRecoveredErrorCount, that.myRecoveredErrorCount).isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37).appendSuper(super.hashCode()).append(myRecordsProcessed).append(myRecoveredErrorCount).toHashCode(); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/BatchWorkChunk.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java similarity index 70% rename from hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/BatchWorkChunk.java rename to hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java index e3f7dd13e32e..c239495cb958 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/coordinator/BatchWorkChunk.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkCreateEvent.java @@ -1,4 +1,4 @@ -package ca.uhn.fhir.batch2.coordinator; +package ca.uhn.fhir.batch2.model; /*- * #%L @@ -20,15 +20,18 @@ * #L% */ -import ca.uhn.fhir.batch2.model.JobDefinition; import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; import javax.annotation.Nonnull; import javax.annotation.Nullable; -public class BatchWorkChunk { - +/** + * The data required for the create transition. + * Payload for the work-chunk creation event including all the job coordinates, the chunk data, and a sequence within the step. + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ +public class WorkChunkCreateEvent { public final String jobDefinitionId; public final int jobDefinitionVersion; public final String targetStepId; @@ -45,8 +48,7 @@ public class BatchWorkChunk { * @param theInstanceId The instance ID associated with this chunk * @param theSerializedData The data. This will be in the form of a map where the values may be strings, lists, and other maps (i.e. JSON) */ - - public BatchWorkChunk(@Nonnull String theJobDefinitionId, int theJobDefinitionVersion, @Nonnull String theTargetStepId, @Nonnull String theInstanceId, int theSequence, @Nullable String theSerializedData) { + public WorkChunkCreateEvent(@Nonnull String theJobDefinitionId, int theJobDefinitionVersion, @Nonnull String theTargetStepId, @Nonnull String theInstanceId, int theSequence, @Nullable String theSerializedData) { jobDefinitionId = theJobDefinitionId; jobDefinitionVersion = theJobDefinitionVersion; targetStepId = theTargetStepId; @@ -55,11 +57,11 @@ public BatchWorkChunk(@Nonnull String theJobDefinitionId, int theJobDefinitionVe serializedData = theSerializedData; } - public static BatchWorkChunk firstChunk(JobDefinition theJobDefinition, String theInstanceId) { + public static WorkChunkCreateEvent firstChunk(JobDefinition theJobDefinition, String theInstanceId) { String firstStepId = theJobDefinition.getFirstStepId(); String jobDefinitionId = theJobDefinition.getJobDefinitionId(); int jobDefinitionVersion = theJobDefinition.getJobDefinitionVersion(); - return new BatchWorkChunk(jobDefinitionId, jobDefinitionVersion, firstStepId, theInstanceId, 0, null); + return new WorkChunkCreateEvent(jobDefinitionId, jobDefinitionVersion, firstStepId, theInstanceId, 0, null); } @Override @@ -68,20 +70,27 @@ public boolean equals(Object theO) { if (theO == null || getClass() != theO.getClass()) return false; - BatchWorkChunk that = (BatchWorkChunk) theO; + WorkChunkCreateEvent that = (WorkChunkCreateEvent) theO; return new EqualsBuilder() - .append(jobDefinitionVersion, that.jobDefinitionVersion) - .append(sequence, that.sequence) .append(jobDefinitionId, that.jobDefinitionId) + .append(jobDefinitionVersion, that.jobDefinitionVersion) .append(targetStepId, that.targetStepId) .append(instanceId, that.instanceId) + .append(sequence, that.sequence) .append(serializedData, that.serializedData) .isEquals(); } @Override public int hashCode() { - return new HashCodeBuilder(17, 37).append(jobDefinitionId).append(jobDefinitionVersion).append(targetStepId).append(instanceId).append(sequence).append(serializedData).toHashCode(); + return new HashCodeBuilder(17, 37) + .append(jobDefinitionId) + .append(jobDefinitionVersion) + .append(targetStepId) + .append(instanceId) + .append(sequence) + .append(serializedData) + .toHashCode(); } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkErrorEvent.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkErrorEvent.java new file mode 100644 index 000000000000..914d9e3192e3 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkErrorEvent.java @@ -0,0 +1,76 @@ +package ca.uhn.fhir.batch2.model; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + + +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.apache.commons.lang3.builder.HashCodeBuilder; + +/** + * Payload for the work-chunk error event including the error message, and the allowed retry count. + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ +public class WorkChunkErrorEvent extends BaseWorkChunkEvent { + + private String myErrorMsg; + + public WorkChunkErrorEvent(String theChunkId) { + super(theChunkId); + } + + public WorkChunkErrorEvent(String theChunkId, String theErrorMessage) { + super(theChunkId); + myErrorMsg = theErrorMessage; + } + + public String getErrorMsg() { + return myErrorMsg; + } + + public WorkChunkErrorEvent setErrorMsg(String theErrorMsg) { + myErrorMsg = theErrorMsg; + return this; + } + + @Override + public boolean equals(Object theO) { + if (this == theO) return true; + + if (theO == null || getClass() != theO.getClass()) return false; + + WorkChunkErrorEvent that = (WorkChunkErrorEvent) theO; + + return new EqualsBuilder() + .appendSuper(super.equals(theO)) + .append(myChunkId, that.myChunkId) + .append(myErrorMsg, that.myErrorMsg) + .isEquals(); + } + + @Override + public int hashCode() { + return new HashCodeBuilder(17, 37) + .appendSuper(super.hashCode()) + .append(myChunkId) + .append(myErrorMsg) + .toHashCode(); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java new file mode 100644 index 000000000000..620da0591aaf --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnum.java @@ -0,0 +1,72 @@ +package ca.uhn.fhir.batch2.model; + +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import java.util.EnumMap; +import java.util.EnumSet; +import java.util.Set; + +/** + * States for the {@link WorkChunk} state machine. + * @see hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md + */ +public enum WorkChunkStatusEnum { + // TODO MB: missing a state - WAITING for gated. it would simplify stats - not in this MR - later + QUEUED, IN_PROGRESS, ERRORED, FAILED, COMPLETED; + + private static final EnumMap> ourPriorStates; + static { + ourPriorStates = new EnumMap<>(WorkChunkStatusEnum.class); + for (WorkChunkStatusEnum nextEnum: WorkChunkStatusEnum.values()) { + ourPriorStates.put(nextEnum, EnumSet.noneOf(WorkChunkStatusEnum.class)); + } + for (WorkChunkStatusEnum nextPriorEnum: WorkChunkStatusEnum.values()) { + for (WorkChunkStatusEnum nextEnum: nextPriorEnum.getNextStates()) { + ourPriorStates.get(nextEnum).add(nextPriorEnum); + } + } + } + + + public boolean isIncomplete() { + return (this != WorkChunkStatusEnum.COMPLETED); + } + + public Set getNextStates() { + switch (this) { + case QUEUED: + return EnumSet.of(IN_PROGRESS); + case IN_PROGRESS: + return EnumSet.of(IN_PROGRESS, ERRORED, FAILED, COMPLETED); + case ERRORED: + return EnumSet.of(IN_PROGRESS, FAILED, COMPLETED); + // terminal states + case FAILED: + case COMPLETED: + default: + return EnumSet.noneOf(WorkChunkStatusEnum.class); + } + } + + public Set getPriorStates() { + return ourPriorStates.get(this); + } +} diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java new file mode 100644 index 000000000000..1a25ce4595b3 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/package-info.java @@ -0,0 +1,39 @@ +/*- + * #%L + * HAPI FHIR JPA Server - Batch2 Task Processor + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +/** + * Our distributed batch processing library. + * + * A running job corresponds to a {@link ca.uhn.fhir.batch2.model.JobInstance}. + * Jobs are modeled as a sequence of steps, operating on {@link ca.uhn.fhir.batch2.model.WorkChunk}s + * containing json data. The first step is special -- it is empty, and the data is assumed to be the job parameters. + * A {@link ca.uhn.fhir.batch2.model.JobDefinition} defines the sequence of {@link ca.uhn.fhir.batch2.model.JobDefinitionStep}s. + * Each step defines the input chunk type, the output chunk type, and a procedure that receives the input and emits 0 or more outputs. + * We have a special kind of final step called a reducer, which corresponds to the stream Collector concept. + * + * Design gaps: + *

    + *
  • If the maintenance job is killed while sending notifications about + * a gated step advance, remaining chunks will never be notified. A CREATED state before QUEUED would catch this. + *
    • + *
+ */ +package ca.uhn.fhir.batch2; + diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java index dc4a65e0bada..96cede0048a0 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/InstanceProgress.java @@ -23,6 +23,7 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.util.StopWatch; import org.apache.commons.lang3.builder.ToStringBuilder; @@ -33,21 +34,23 @@ import java.util.Map; import java.util.concurrent.TimeUnit; -class InstanceProgress { +public class InstanceProgress { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); private int myRecordsProcessed = 0; + + // these 4 cover all chunks private int myIncompleteChunkCount = 0; - private int myQueuedCount = 0; private int myCompleteChunkCount = 0; private int myErroredChunkCount = 0; private int myFailedChunkCount = 0; + private int myErrorCountForAllStatuses = 0; - private Long myEarliestStartTime = null; - private Long myLatestEndTime = null; + private Date myEarliestStartTime = null; + private Date myLatestEndTime = null; private String myErrormessage = null; private StatusEnum myNewStatus = null; - private Map> myStepToStatusCountMap = new HashMap<>(); + private final Map> myStepToStatusCountMap = new HashMap<>(); public void addChunk(WorkChunk theChunk) { myErrorCountForAllStatuses += theChunk.getErrorCount(); @@ -60,7 +63,7 @@ public void addChunk(WorkChunk theChunk) { private void updateCompletionStatus(WorkChunk theChunk) { //Update the status map first. - Map statusToCountMap = myStepToStatusCountMap.getOrDefault(theChunk.getTargetStepId(), new HashMap<>()); + Map statusToCountMap = myStepToStatusCountMap.getOrDefault(theChunk.getTargetStepId(), new HashMap<>()); statusToCountMap.put(theChunk.getStatus(), statusToCountMap.getOrDefault(theChunk.getStatus(), 0) + 1); switch (theChunk.getStatus()) { @@ -81,25 +84,19 @@ private void updateCompletionStatus(WorkChunk theChunk) { myFailedChunkCount++; myErrormessage = theChunk.getErrorMessage(); break; - case CANCELLED: - break; } ourLog.trace("Chunk has status {} with errored chunk count {}", theChunk.getStatus(), myErroredChunkCount); } private void updateLatestEndTime(WorkChunk theChunk) { - if (theChunk.getEndTime() != null) { - if (myLatestEndTime == null || myLatestEndTime < theChunk.getEndTime().getTime()) { - myLatestEndTime = theChunk.getEndTime().getTime(); - } + if (theChunk.getEndTime() != null && (myLatestEndTime == null || myLatestEndTime.before(theChunk.getEndTime()))) { + myLatestEndTime = theChunk.getEndTime(); } } private void updateEarliestTime(WorkChunk theChunk) { - if (theChunk.getStartTime() != null) { - if (myEarliestStartTime == null || myEarliestStartTime > theChunk.getStartTime().getTime()) { - myEarliestStartTime = theChunk.getStartTime().getTime(); - } + if (theChunk.getStartTime() != null && (myEarliestStartTime == null || myEarliestStartTime.after(theChunk.getStartTime()))) { + myEarliestStartTime = theChunk.getStartTime(); } } @@ -109,67 +106,68 @@ private void updateRecordsProcessed(WorkChunk theChunk) { } } + /** + * Update the job instance with status information. + * We shouldn't read any values from theInstance here -- just write. + * + * @param theInstance the instance to update with progress statistics + */ public void updateInstance(JobInstance theInstance) { if (myEarliestStartTime != null) { - theInstance.setStartTime(new Date(myEarliestStartTime)); + theInstance.setStartTime(myEarliestStartTime); + } + if (myLatestEndTime != null && hasNewStatus() && myNewStatus.isEnded()) { + theInstance.setEndTime(myLatestEndTime); } theInstance.setErrorCount(myErrorCountForAllStatuses); theInstance.setCombinedRecordsProcessed(myRecordsProcessed); - updateStatus(theInstance); - - setEndTime(theInstance); - - theInstance.setErrorMessage(myErrormessage); - } - - private void setEndTime(JobInstance theInstance) { - if (myLatestEndTime != null) { - if (myFailedChunkCount > 0) { - theInstance.setEndTime(new Date(myLatestEndTime)); - } else if (myCompleteChunkCount > 0 && myIncompleteChunkCount == 0 && myErroredChunkCount == 0) { - theInstance.setEndTime(new Date(myLatestEndTime)); - } + if (getChunkCount() > 0) { + double percentComplete = (double) (myCompleteChunkCount) / (double) getChunkCount(); + theInstance.setProgress(percentComplete); } - } - - private void updateStatus(JobInstance theInstance) { - ourLog.trace("Updating status for instance with errors: {}", myErroredChunkCount); - if (myCompleteChunkCount >= 1 || myErroredChunkCount >= 1) { - double percentComplete = (double) (myCompleteChunkCount) / (double) (myIncompleteChunkCount + myCompleteChunkCount + myFailedChunkCount + myErroredChunkCount); - theInstance.setProgress(percentComplete); + if (myEarliestStartTime != null && myLatestEndTime != null) { + long elapsedTime = myLatestEndTime.getTime() - myEarliestStartTime.getTime(); + if (elapsedTime > 0) { + double throughput = StopWatch.getThroughput(myRecordsProcessed, elapsedTime, TimeUnit.SECONDS); + theInstance.setCombinedRecordsProcessedPerSecond(throughput); - if (jobSuccessfullyCompleted()) { - myNewStatus = StatusEnum.COMPLETED; - } else if (myErroredChunkCount > 0) { - myNewStatus = StatusEnum.ERRORED; + String estimatedTimeRemaining = StopWatch.formatEstimatedTimeRemaining(myCompleteChunkCount, getChunkCount(), elapsedTime); + theInstance.setEstimatedTimeRemaining(estimatedTimeRemaining); } + } - ourLog.trace("Status is now {} with errored chunk count {}", myNewStatus, myErroredChunkCount); - if (myEarliestStartTime != null && myLatestEndTime != null) { - long elapsedTime = myLatestEndTime - myEarliestStartTime; - if (elapsedTime > 0) { - double throughput = StopWatch.getThroughput(myRecordsProcessed, elapsedTime, TimeUnit.SECONDS); - theInstance.setCombinedRecordsProcessedPerSecond(throughput); + theInstance.setErrorMessage(myErrormessage); - String estimatedTimeRemaining = StopWatch.formatEstimatedTimeRemaining(myCompleteChunkCount, (myCompleteChunkCount + myIncompleteChunkCount), elapsedTime); - theInstance.setEstimatedTimeRemaining(estimatedTimeRemaining); - } - } + if (hasNewStatus()) { + ourLog.trace("Status will change for {}: {}", theInstance.getInstanceId(), myNewStatus); } + + ourLog.trace("Updating status for instance with errors: {}", myErroredChunkCount); + ourLog.trace("Statistics for job {}: complete/in-progress/errored/failed chunk count {}/{}/{}/{}", + theInstance.getInstanceId(), myCompleteChunkCount, myIncompleteChunkCount, myErroredChunkCount, myFailedChunkCount); } - private boolean jobSuccessfullyCompleted() { - return myIncompleteChunkCount == 0 && myErroredChunkCount == 0 && myFailedChunkCount == 0; + private int getChunkCount() { + return myIncompleteChunkCount + myCompleteChunkCount + myFailedChunkCount + myErroredChunkCount; } - public boolean failed() { - return myFailedChunkCount > 0; + /** + * Transitions from IN_PROGRESS/ERRORED based on chunk statuses. + */ + public void calculateNewStatus() { + if (myFailedChunkCount > 0) { + myNewStatus = StatusEnum.FAILED; + } else if (myErroredChunkCount > 0) { + myNewStatus = StatusEnum.ERRORED; + } else if (myIncompleteChunkCount == 0 && myCompleteChunkCount > 0) { + myNewStatus = StatusEnum.COMPLETED; + } } public boolean changed() { - return (myIncompleteChunkCount + myCompleteChunkCount + myErroredChunkCount) >= 2 || myErrorCountForAllStatuses > 0; + return (myIncompleteChunkCount + myCompleteChunkCount + myErroredChunkCount + myErrorCountForAllStatuses) > 0; } @Override diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java index 7bd2162e8211..66a5982ebf18 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceProgressCalculator.java @@ -1,5 +1,3 @@ -package ca.uhn.fhir.batch2.progress; - /*- * #%L * HAPI FHIR JPA Server - Batch2 Task Processor @@ -19,11 +17,11 @@ * limitations under the License. * #L% */ +package ca.uhn.fhir.batch2.progress; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; import ca.uhn.fhir.batch2.maintenance.JobChunkProgressAccumulator; -import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; import ca.uhn.fhir.util.Logs; @@ -42,63 +40,53 @@ public class JobInstanceProgressCalculator { public JobInstanceProgressCalculator(IJobPersistence theJobPersistence, JobChunkProgressAccumulator theProgressAccumulator, JobDefinitionRegistry theJobDefinitionRegistry) { myJobPersistence = theJobPersistence; myProgressAccumulator = theProgressAccumulator; - myJobInstanceStatusUpdater = new JobInstanceStatusUpdater(theJobPersistence, theJobDefinitionRegistry); + myJobInstanceStatusUpdater = new JobInstanceStatusUpdater(theJobDefinitionRegistry); } - public void calculateAndStoreInstanceProgress(JobInstance theInstance) { - String instanceId = theInstance.getInstanceId(); + public void calculateAndStoreInstanceProgress(String theInstanceId) { StopWatch stopWatch = new StopWatch(); - ourLog.trace("calculating progress: {}", instanceId); - + ourLog.trace("calculating progress: {}", theInstanceId); - InstanceProgress instanceProgress = calculateInstanceProgress(instanceId); + InstanceProgress instanceProgress = calculateInstanceProgress(theInstanceId); - if (instanceProgress.failed()) { - myJobInstanceStatusUpdater.setFailed(theInstance); - } - - JobInstance currentInstance = myJobPersistence.fetchInstance(instanceId).orElse(null); - if (currentInstance != null) { + myJobPersistence.updateInstance(theInstanceId, currentInstance->{ instanceProgress.updateInstance(currentInstance); if (instanceProgress.changed() || currentInstance.getStatus() == StatusEnum.IN_PROGRESS) { if (currentInstance.getCombinedRecordsProcessed() > 0) { ourLog.info("Job {} of type {} has status {} - {} records processed ({}/sec) - ETA: {}", currentInstance.getInstanceId(), currentInstance.getJobDefinitionId(), currentInstance.getStatus(), currentInstance.getCombinedRecordsProcessed(), currentInstance.getCombinedRecordsProcessedPerSecond(), currentInstance.getEstimatedTimeRemaining()); - ourLog.debug(instanceProgress.toString()); } else { ourLog.info("Job {} of type {} has status {} - {} records processed", currentInstance.getInstanceId(), currentInstance.getJobDefinitionId(), currentInstance.getStatus(), currentInstance.getCombinedRecordsProcessed()); - ourLog.debug(instanceProgress.toString()); } + ourLog.debug(instanceProgress.toString()); } - if (instanceProgress.changed()) { - if (instanceProgress.hasNewStatus()) { - myJobInstanceStatusUpdater.updateInstanceStatus(currentInstance, instanceProgress.getNewStatus()); - } else { - myJobPersistence.updateInstance(currentInstance); - } + + if (instanceProgress.hasNewStatus()) { + myJobInstanceStatusUpdater.updateInstanceStatus(currentInstance, instanceProgress.getNewStatus()); } - } - ourLog.trace("calculating progress: {} - complete in {}", instanceId, stopWatch); + return true; + }); + ourLog.trace("calculating progress: {} - complete in {}", theInstanceId, stopWatch); } @Nonnull - private InstanceProgress calculateInstanceProgress(String instanceId) { + public InstanceProgress calculateInstanceProgress(String instanceId) { InstanceProgress instanceProgress = new InstanceProgress(); - // wipmb mb here Iterator workChunkIterator = myJobPersistence.fetchAllWorkChunksIterator(instanceId, false); while (workChunkIterator.hasNext()) { WorkChunk next = workChunkIterator.next(); + // global stats myProgressAccumulator.addChunk(next); + // instance stats instanceProgress.addChunk(next); } + + instanceProgress.calculateNewStatus(); + return instanceProgress; } - public void calculateInstanceProgressAndPopulateInstance(JobInstance theInstance) { - InstanceProgress progress = calculateInstanceProgress(theInstance.getInstanceId()); - progress.updateInstance(theInstance); - } } diff --git a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdater.java b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdater.java index 17f1ba76a66d..e98d5bddfd25 100644 --- a/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdater.java +++ b/hapi-fhir-storage-batch2/src/main/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdater.java @@ -21,28 +21,29 @@ */ import ca.uhn.fhir.batch2.api.IJobCompletionHandler; -import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobCompletionDetails; import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.StatusEnum; -import ca.uhn.fhir.util.Logs; import ca.uhn.fhir.model.api.IModelJson; +import ca.uhn.fhir.util.Logs; import org.slf4j.Logger; -import java.util.Optional; - public class JobInstanceStatusUpdater { private static final Logger ourLog = Logs.getBatchTroubleshootingLog(); - private final IJobPersistence myJobPersistence; private final JobDefinitionRegistry myJobDefinitionRegistry; - public JobInstanceStatusUpdater(IJobPersistence theJobPersistence, JobDefinitionRegistry theJobDefinitionRegistry) { - myJobPersistence = theJobPersistence; + public JobInstanceStatusUpdater(JobDefinitionRegistry theJobDefinitionRegistry) { myJobDefinitionRegistry = theJobDefinitionRegistry; } + /** + * Update the status on the instance, and call any completion handlers when entering a completion state. + * @param theJobInstance the instance to mutate + * @param theNewStatus target status + * @return was the state change allowed? + */ public boolean updateInstanceStatus(JobInstance theJobInstance, StatusEnum theNewStatus) { StatusEnum origStatus = theJobInstance.getStatus(); if (origStatus == theNewStatus) { @@ -54,34 +55,9 @@ public boolean updateInstanceStatus(JobInstance theJobInstance, StatusEnum theNe } theJobInstance.setStatus(theNewStatus); ourLog.debug("Updating job instance {} of type {} from {} to {}", theJobInstance.getInstanceId(), theJobInstance.getJobDefinitionId(), origStatus, theNewStatus); - return updateInstance(theJobInstance); - } - - private boolean updateInstance(JobInstance theJobInstance) { - Optional oInstance = myJobPersistence.fetchInstance(theJobInstance.getInstanceId()); - if (oInstance.isEmpty()) { - ourLog.error("Trying to update instance of non-existent Instance {}", theJobInstance); - return false; - } - - StatusEnum origStatus = oInstance.get().getStatus(); - StatusEnum newStatus = theJobInstance.getStatus(); - if (!StatusEnum.isLegalStateTransition(origStatus, newStatus)) { - ourLog.error("Ignoring illegal state transition for job instance {} of type {} from {} to {}", theJobInstance.getInstanceId(), theJobInstance.getJobDefinitionId(), origStatus, newStatus); - return false; - } + handleStatusChange(theJobInstance); - boolean statusChanged = myJobPersistence.updateInstance(theJobInstance); - - // This code can be called by both the maintenance service and the fast track work step executor. - // We only want to call the completion handler if the status was changed to COMPLETED in this thread. We use the - // record changed count from of a sql update change status to rely on the database to tell us which thread - // the status change happened in. - if (statusChanged) { - ourLog.info("Changing job instance {} of type {} from {} to {}", theJobInstance.getInstanceId(), theJobInstance.getJobDefinitionId(), origStatus, theJobInstance.getStatus()); - handleStatusChange(theJobInstance); - } - return statusChanged; + return true; } private void handleStatusChange(JobInstance theJobInstance) { @@ -114,19 +90,4 @@ private void invokeCompletionHandler(JobInstance theJobI theJobCompletionHandler.jobComplete(completionDetails); } - public boolean setCompleted(JobInstance theInstance) { - return updateInstanceStatus(theInstance, StatusEnum.COMPLETED); - } - - public boolean setInProgress(JobInstance theInstance) { - return updateInstanceStatus(theInstance, StatusEnum.IN_PROGRESS); - } - - public boolean setCancelled(JobInstance theInstance) { - return updateInstanceStatus(theInstance, StatusEnum.CANCELLED); - } - - public boolean setFailed(JobInstance theInstance) { - return updateInstanceStatus(theInstance, StatusEnum.FAILED); - } } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java index 0a0db91bc2ef..3d63aa363c27 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobCoordinatorImplTest.java @@ -15,9 +15,11 @@ import ca.uhn.fhir.batch2.model.JobInstanceStartRequest; import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.JobWorkNotificationJsonMessage; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.batch.models.Batch2JobStartResponse; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; @@ -45,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; @@ -71,7 +73,6 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { @Mock private IJobMaintenanceService myJobMaintenanceService; private IHapiTransactionService myTransactionService = new NonTransactionalHapiTransactionService(); - @Captor private ArgumentCaptor> myStep1ExecutionDetailsCaptor; @Captor @@ -82,16 +83,18 @@ public class JobCoordinatorImplTest extends BaseBatch2Test { private ArgumentCaptor myJobWorkNotificationCaptor; @Captor private ArgumentCaptor myJobInstanceCaptor; + @Captor + private ArgumentCaptor myJobDefinitionCaptor; + @Captor + private ArgumentCaptor myParametersJsonCaptor; - private final JobInstance ourQueuedInstance = createInstance(JOB_DEFINITION_ID, StatusEnum.QUEUED); @BeforeEach public void beforeEach() { // The code refactored to keep the same functionality, // but in this service (so it's a real service here!) - WorkChunkProcessor jobStepExecutorSvc = new WorkChunkProcessor(myJobInstancePersister, myBatchJobSender, myTransactionService); - JobStepExecutorFactory jobStepExecutorFactory = new JobStepExecutorFactory(myJobInstancePersister, myBatchJobSender, jobStepExecutorSvc, myJobMaintenanceService, myJobDefinitionRegistry); - mySvc = new JobCoordinatorImpl(myBatchJobSender, myWorkChannelReceiver, myJobInstancePersister, myJobDefinitionRegistry, jobStepExecutorSvc, myJobMaintenanceService); + WorkChunkProcessor jobStepExecutorSvc = new WorkChunkProcessor(myJobInstancePersister, myBatchJobSender); + mySvc = new JobCoordinatorImpl(myBatchJobSender, myWorkChannelReceiver, myJobInstancePersister, myJobDefinitionRegistry, jobStepExecutorSvc, myJobMaintenanceService, myTransactionService); } @AfterEach @@ -139,7 +142,7 @@ public void testPerformStep_FirstStep() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), any(), eq(50)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 50, 0)); verify(myJobInstancePersister, times(0)).fetchWorkChunksWithoutData(any(), anyInt(), anyInt()); verify(myBatchJobSender, times(2)).sendWorkChannelMessage(any()); } @@ -147,7 +150,7 @@ public void testPerformStep_FirstStep() { private void setupMocks(JobDefinition theJobDefinition, WorkChunk theWorkChunk) { mockJobRegistry(theJobDefinition); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(theWorkChunk)); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(theWorkChunk)); } private void mockJobRegistry(JobDefinition theJobDefinition) { @@ -176,8 +179,6 @@ public void startInstance_usingExistingCache_returnsExistingIncompleteJobFirst() existingCompletedInstance.setInstanceId(completedInstanceId); // when - when(myJobDefinitionRegistry.getLatestJobDefinition(eq(JOB_DEFINITION_ID))) - .thenReturn(Optional.of(def)); when(myJobInstancePersister.fetchInstances(any(FetchJobInstancesRequest.class), anyInt(), anyInt())) .thenReturn(Arrays.asList(existingInProgInstance)); @@ -198,37 +199,6 @@ public void startInstance_usingExistingCache_returnsExistingIncompleteJobFirst() ); } - /** - * If the first step doesn't produce any work chunks, then - * the instance should be marked as complete right away. - */ - @Test - public void testPerformStep_FirstStep_NoWorkChunksProduced() { - - // Setup - - setupMocks(createJobDefinition(), createWorkChunkStep1()); - when(myStep1Worker.run(any(), any())).thenReturn(new RunOutcome(50)); - when(myJobInstancePersister.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(ourQueuedInstance)); - - mySvc.start(); - - // Execute - - myWorkChannelReceiver.send(new JobWorkNotificationJsonMessage(createWorkNotification(STEP_1))); - - // Verify - - verify(myStep1Worker, times(1)).run(myStep1ExecutionDetailsCaptor.capture(), any()); - TestJobParameters params = myStep1ExecutionDetailsCaptor.getValue().getParameters(); - assertEquals(PARAM_1_VALUE, params.getParam1()); - assertEquals(PARAM_2_VALUE, params.getParam2()); - assertEquals(PASSWORD_VALUE, params.getPassword()); - - // QUEUED -> IN_PROGRESS and IN_PROGRESS -> COMPLETED - verify(myJobInstancePersister, times(2)).updateInstance(any()); - } - @Test public void testPerformStep_FirstStep_GatedExecutionMode() { @@ -257,7 +227,7 @@ public void testPerformStep_FirstStep_GatedExecutionMode() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), any(), eq(50)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 50, 0)); verify(myBatchJobSender, times(0)).sendWorkChannelMessage(any()); } @@ -266,7 +236,7 @@ public void testPerformStep_SecondStep() { // Setup - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); doReturn(createJobDefinition()).when(myJobDefinitionRegistry).getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1)); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); when(myStep2Worker.run(any(), any())).thenReturn(new RunOutcome(50)); @@ -284,7 +254,7 @@ public void testPerformStep_SecondStep() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), eq(CHUNK_ID), eq(50)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 50, 0)); } @Test @@ -293,7 +263,7 @@ public void testPerformStep_SecondStep_WorkerFailure() { // Setup AtomicInteger counter = new AtomicInteger(); doReturn(createJobDefinition()).when(myJobDefinitionRegistry).getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1)); - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); when(myStep2Worker.run(any(), any())).thenAnswer(t->{ if (counter.getAndIncrement() == 0) { @@ -316,13 +286,13 @@ public void testPerformStep_SecondStep_WorkerFailure() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - ArgumentCaptor parametersArgumentCaptor = ArgumentCaptor.forClass(MarkWorkChunkAsErrorRequest.class); - verify(myJobInstancePersister, times(1)).markWorkChunkAsErroredAndIncrementErrorCount(parametersArgumentCaptor.capture()); - MarkWorkChunkAsErrorRequest capturedParams = parametersArgumentCaptor.getValue(); + ArgumentCaptor parametersArgumentCaptor = ArgumentCaptor.forClass(WorkChunkErrorEvent.class); + verify(myJobInstancePersister, times(1)).onWorkChunkError(parametersArgumentCaptor.capture()); + WorkChunkErrorEvent capturedParams = parametersArgumentCaptor.getValue(); assertEquals(CHUNK_ID, capturedParams.getChunkId()); assertEquals("This is an error message", capturedParams.getErrorMsg()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), eq(CHUNK_ID), eq(0)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 0, 0)); } @@ -331,7 +301,7 @@ public void testPerformStep_SecondStep_WorkerReportsRecoveredErrors() { // Setup - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_2, new TestJobStep2InputType(DATA_1_VALUE, DATA_2_VALUE)))); doReturn(createJobDefinition()).when(myJobDefinitionRegistry).getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1)); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); when(myStep2Worker.run(any(), any())).thenAnswer(t -> { @@ -354,8 +324,7 @@ public void testPerformStep_SecondStep_WorkerReportsRecoveredErrors() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - verify(myJobInstancePersister, times(1)).incrementWorkChunkErrorCount(eq(CHUNK_ID), eq(2)); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), eq(CHUNK_ID), eq(50)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(eq(new WorkChunkCompletionEvent(CHUNK_ID, 50, 2))); } @Test @@ -363,7 +332,7 @@ public void testPerformStep_FinalStep() { // Setup - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunkStep3())); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunkStep3())); doReturn(createJobDefinition()).when(myJobDefinitionRegistry).getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1)); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); when(myStep3Worker.run(any(), any())).thenReturn(new RunOutcome(50)); @@ -381,7 +350,7 @@ public void testPerformStep_FinalStep() { assertEquals(PARAM_2_VALUE, params.getParam2()); assertEquals(PASSWORD_VALUE, params.getPassword()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsCompletedAndClearData(eq(INSTANCE_ID), eq(CHUNK_ID), eq(50)); + verify(myJobInstancePersister, times(1)).onWorkChunkCompletion(new WorkChunkCompletionEvent(CHUNK_ID, 50, 0)); } @SuppressWarnings("unchecked") @@ -390,7 +359,7 @@ public void testPerformStep_FinalStep_PreventChunkWriting() { // Setup - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_3, new TestJobStep3InputType().setData3(DATA_3_VALUE).setData4(DATA_4_VALUE)))); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunk(STEP_3, new TestJobStep3InputType().setData3(DATA_3_VALUE).setData4(DATA_4_VALUE)))); doReturn(createJobDefinition()).when(myJobDefinitionRegistry).getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1)); when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); when(myStep3Worker.run(any(), any())).thenAnswer(t -> { @@ -407,7 +376,7 @@ public void testPerformStep_FinalStep_PreventChunkWriting() { // Verify verify(myStep3Worker, times(1)).run(myStep3ExecutionDetailsCaptor.capture(), any()); - verify(myJobInstancePersister, times(1)).markWorkChunkAsFailed(eq(CHUNK_ID), any()); + verify(myJobInstancePersister, times(1)).onWorkChunkFailed(eq(CHUNK_ID), any()); } @Test @@ -417,7 +386,6 @@ public void testPerformStep_DefinitionNotKnown() { String exceptionMessage = "badbadnotgood"; when(myJobDefinitionRegistry.getJobDefinitionOrThrowException(eq(JOB_DEFINITION_ID), eq(1))).thenThrow(new JobExecutionFailedException(exceptionMessage)); - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.of(createWorkChunkStep2())); mySvc.start(); // Execute @@ -442,8 +410,10 @@ public void testPerformStep_DefinitionNotKnown() { public void testPerformStep_ChunkNotKnown() { // Setup - - when(myJobInstancePersister.fetchWorkChunkSetStartTimeAndMarkInProgress(eq(CHUNK_ID))).thenReturn(Optional.empty()); + JobDefinition jobDefinition = createJobDefinition(); + when(myJobDefinitionRegistry.getJobDefinitionOrThrowException(JOB_DEFINITION_ID, 1)).thenReturn(jobDefinition); + when(myJobInstancePersister.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); + when(myJobInstancePersister.onWorkChunkDequeue(eq(CHUNK_ID))).thenReturn(Optional.empty()); mySvc.start(); // Execute @@ -487,10 +457,10 @@ public void testStartInstance() { // Setup + JobDefinition jobDefinition = createJobDefinition(); when(myJobDefinitionRegistry.getLatestJobDefinition(eq(JOB_DEFINITION_ID))) - .thenReturn(Optional.of(createJobDefinition())); - when(myJobInstancePersister.storeNewInstance(any())) - .thenReturn(INSTANCE_ID).thenReturn(INSTANCE_ID); + .thenReturn(Optional.of(jobDefinition)); + when(myJobInstancePersister.onCreateWithFirstChunk(any(), any())).thenReturn(new IJobPersistence.CreateResult(INSTANCE_ID, CHUNK_ID)); // Execute @@ -502,24 +472,16 @@ public void testStartInstance() { // Verify verify(myJobInstancePersister, times(1)) - .storeNewInstance(myJobInstanceCaptor.capture()); - assertNull(myJobInstanceCaptor.getValue().getInstanceId()); - assertEquals(JOB_DEFINITION_ID, myJobInstanceCaptor.getValue().getJobDefinitionId()); - assertEquals(1, myJobInstanceCaptor.getValue().getJobDefinitionVersion()); - assertEquals(PARAM_1_VALUE, myJobInstanceCaptor.getValue().getParameters(TestJobParameters.class).getParam1()); - assertEquals(PARAM_2_VALUE, myJobInstanceCaptor.getValue().getParameters(TestJobParameters.class).getParam2()); - assertEquals(PASSWORD_VALUE, myJobInstanceCaptor.getValue().getParameters(TestJobParameters.class).getPassword()); - assertEquals(StatusEnum.QUEUED, myJobInstanceCaptor.getValue().getStatus()); + .onCreateWithFirstChunk(myJobDefinitionCaptor.capture(), myParametersJsonCaptor.capture()); + assertSame(jobDefinition, myJobDefinitionCaptor.getValue()); + assertEquals(startRequest.getParameters(), myParametersJsonCaptor.getValue()); verify(myBatchJobSender, times(1)).sendWorkChannelMessage(myJobWorkNotificationCaptor.capture()); - assertNull(myJobWorkNotificationCaptor.getAllValues().get(0).getChunkId()); + assertEquals(CHUNK_ID, myJobWorkNotificationCaptor.getAllValues().get(0).getChunkId()); assertEquals(JOB_DEFINITION_ID, myJobWorkNotificationCaptor.getAllValues().get(0).getJobDefinitionId()); assertEquals(1, myJobWorkNotificationCaptor.getAllValues().get(0).getJobDefinitionVersion()); assertEquals(STEP_1, myJobWorkNotificationCaptor.getAllValues().get(0).getTargetStepId()); - BatchWorkChunk expectedWorkChunk = new BatchWorkChunk(JOB_DEFINITION_ID, 1, STEP_1, INSTANCE_ID, 0, null); - verify(myJobInstancePersister, times(1)).storeWorkChunk(eq(expectedWorkChunk)); - verifyNoMoreInteractions(myJobInstancePersister); verifyNoMoreInteractions(myStep1Worker); verifyNoMoreInteractions(myStep2Worker); @@ -613,7 +575,7 @@ static WorkChunk createWorkChunk(String theJobId, String theTargetStepId, IModel .setJobDefinitionVersion(1) .setTargetStepId(theTargetStepId) .setData(theData) - .setStatus(StatusEnum.IN_PROGRESS) + .setStatus(WorkChunkStatusEnum.IN_PROGRESS) .setInstanceId(INSTANCE_ID); } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java index 97960fc22d72..1fb4274bcbeb 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/JobDataSinkTest.java @@ -13,6 +13,7 @@ import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.JobWorkNotification; +import ca.uhn.fhir.batch2.model.WorkChunkCreateEvent; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.JsonUtil; import com.fasterxml.jackson.annotation.JsonProperty; @@ -52,7 +53,7 @@ class JobDataSinkTest { @Captor private ArgumentCaptor myJobWorkNotificationCaptor; @Captor - private ArgumentCaptor myBatchWorkChunkCaptor; + private ArgumentCaptor myBatchWorkChunkCaptor; @Test public void test_sink_accept() { @@ -93,7 +94,7 @@ public RunOutcome run(@Nonnull StepExecutionDetails details = new StepExecutionDetails<>(new TestJobParameters().setParam1("" + PID_COUNT), null, instance, CHUNK_ID); JobWorkCursor cursor = new JobWorkCursor<>(job, true, firstStep, lastStep); @@ -115,7 +116,7 @@ public RunOutcome run(@Nonnull StepExecutionDetails chunkData = new WorkChunkData<>(stepData); // when - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))) - .thenReturn(Optional.of(JobInstance.fromInstanceId(INSTANCE_ID))); + JobInstance instance = JobInstance.fromInstanceId(INSTANCE_ID); + instance.setStatus(StatusEnum.FINALIZE); + stubUpdateInstanceCallback(instance); when(myJobPersistence.fetchAllWorkChunksIterator(any(), anyBoolean())).thenReturn(Collections.emptyIterator()); // test myDataSink.accept(chunkData); // verify - ArgumentCaptor instanceCaptor = ArgumentCaptor.forClass(JobInstance.class); - verify(myJobPersistence) - .updateInstance(instanceCaptor.capture()); - - assertEquals(JsonUtil.serialize(stepData, false), instanceCaptor.getValue().getReport()); + assertEquals(JsonUtil.serialize(stepData, false), instance.getReport()); } @Test @@ -119,23 +114,23 @@ public void accept_multipleCalls_firstInWins() { ourLogger.setLevel(Level.ERROR); - // when - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))) - .thenReturn(Optional.of(JobInstance.fromInstanceId(INSTANCE_ID))); + JobInstance instance = JobInstance.fromInstanceId(INSTANCE_ID); + instance.setStatus(StatusEnum.FINALIZE); when(myJobPersistence.fetchAllWorkChunksIterator(any(), anyBoolean())).thenReturn(Collections.emptyIterator()); + stubUpdateInstanceCallback(instance); // test myDataSink.accept(firstData); - myDataSink.accept(secondData); + assertThrows(IllegalStateException.class, ()-> + myDataSink.accept(secondData)); - // verify - ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ILoggingEvent.class); - verify(myListAppender).doAppend(logCaptor.capture()); - assertEquals(1, logCaptor.getAllValues().size()); - ILoggingEvent log = logCaptor.getValue(); - assertTrue(log.getFormattedMessage().contains( - "Report has already been set. Now it is being overwritten. Last in will win!" - )); + } + + private void stubUpdateInstanceCallback(JobInstance theJobInstance) { + when(myJobPersistence.updateInstance(eq(INSTANCE_ID), any())).thenAnswer(call->{ + IJobPersistence.JobInstanceUpdateCallback callback = call.getArgument(1); + return callback.doUpdate(theJobInstance); + }); } @Test @@ -143,10 +138,8 @@ public void accept_noInstanceIdFound_throwsJobExecutionFailed() { // setup String data = "data"; WorkChunkData chunkData = new WorkChunkData<>(new StepOutputData(data)); - - // when - when(myJobPersistence.fetchInstance(anyString())) - .thenReturn(Optional.empty()); + when(myJobPersistence.updateInstance(any(), any())).thenReturn(false); + when(myJobPersistence.fetchAllWorkChunksIterator(any(), anyBoolean())).thenReturn(Collections.emptyIterator()); // test try { @@ -155,7 +148,7 @@ public void accept_noInstanceIdFound_throwsJobExecutionFailed() { } catch (JobExecutionFailedException ex) { assertTrue(ex.getMessage().contains("No instance found with Id " + INSTANCE_ID)); } catch (Exception anyOtherEx) { - fail(anyOtherEx.getMessage()); + fail("Unexpected exception", anyOtherEx); } } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java index dc09c54851e9..638b80ec31b5 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/ReductionStepExecutorServiceImplTest.java @@ -14,6 +14,7 @@ import ca.uhn.fhir.batch2.model.JobWorkCursor; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; import org.junit.jupiter.api.BeforeEach; @@ -101,7 +102,7 @@ public void doExecution_reductionWithChunkFailed_marksAllFutureChunksAsFailedBut // verification assertFalse(result.isSuccessful()); ArgumentCaptor submittedListIds = ArgumentCaptor.forClass(List.class); - ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(StatusEnum.class); + ArgumentCaptor statusCaptor = ArgumentCaptor.forClass(WorkChunkStatusEnum.class); verify(myJobPersistence, times(chunkIds.size())) .markWorkChunksWithStatusAndWipeData( eq(INSTANCE_ID), @@ -118,9 +119,9 @@ public void doExecution_reductionWithChunkFailed_marksAllFutureChunksAsFailedBut // assumes the order of which is called first // successes, then failures assertEquals(2, statusCaptor.getAllValues().size()); - List statuses = statusCaptor.getAllValues(); - assertEquals(StatusEnum.COMPLETED, statuses.get(0)); - assertEquals(StatusEnum.FAILED, statuses.get(1)); + List statuses = statusCaptor.getAllValues(); + assertEquals(WorkChunkStatusEnum.COMPLETED, statuses.get(0)); + assertEquals(WorkChunkStatusEnum.FAILED, statuses.get(1)); } @@ -165,7 +166,7 @@ public void doExecution_reductionStepWithValidInput_executesAsExpected() { assertTrue(result.isSuccessful()); ArgumentCaptor> chunkIdCaptor = ArgumentCaptor.forClass(List.class); verify(myJobPersistence).markWorkChunksWithStatusAndWipeData(eq(INSTANCE_ID), - chunkIdCaptor.capture(), eq(StatusEnum.COMPLETED), eq(null)); + chunkIdCaptor.capture(), eq(WorkChunkStatusEnum.COMPLETED), eq(null)); List capturedIds = chunkIdCaptor.getValue(); assertEquals(chunkIds.size(), capturedIds.size()); for (String chunkId : chunkIds) { @@ -203,7 +204,7 @@ public void doExecution_reductionStepWithErrors_returnsFalseAndMarksPreviousChun ArgumentCaptor chunkIdCaptor = ArgumentCaptor.forClass(String.class); ArgumentCaptor errorCaptor = ArgumentCaptor.forClass(String.class); verify(myJobPersistence, times(chunkIds.size())) - .markWorkChunkAsFailed(chunkIdCaptor.capture(), errorCaptor.capture()); + .onWorkChunkFailed(chunkIdCaptor.capture(), errorCaptor.capture()); List chunkIdsCaptured = chunkIdCaptor.getAllValues(); List errorsCaptured = errorCaptor.getAllValues(); for (int i = 0; i < chunkIds.size(); i++) { diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java index b649babc692d..83cc133cb95c 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/coordinator/WorkChunkProcessorTest.java @@ -1,6 +1,5 @@ package ca.uhn.fhir.batch2.coordinator; -import ca.uhn.fhir.batch2.api.ChunkExecutionDetails; import ca.uhn.fhir.batch2.api.IJobDataSink; import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.IJobStepWorker; @@ -8,38 +7,30 @@ import ca.uhn.fhir.batch2.api.IReductionStepWorker; import ca.uhn.fhir.batch2.api.JobExecutionFailedException; import ca.uhn.fhir.batch2.api.JobStepFailedException; -import ca.uhn.fhir.batch2.api.ReductionStepExecutionDetails; import ca.uhn.fhir.batch2.api.RunOutcome; import ca.uhn.fhir.batch2.api.StepExecutionDetails; import ca.uhn.fhir.batch2.api.VoidModel; import ca.uhn.fhir.batch2.channel.BatchJobSender; -import ca.uhn.fhir.batch2.model.ChunkOutcome; import ca.uhn.fhir.batch2.model.JobDefinition; import ca.uhn.fhir.batch2.model.JobDefinitionReductionStep; import ca.uhn.fhir.batch2.model.JobDefinitionStep; import ca.uhn.fhir.batch2.model.JobInstance; import ca.uhn.fhir.batch2.model.JobWorkCursor; -import ca.uhn.fhir.batch2.model.MarkWorkChunkAsErrorRequest; -import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkCompletionEvent; import ca.uhn.fhir.batch2.model.WorkChunkData; -import ca.uhn.fhir.jpa.dao.tx.HapiTransactionService; -import ca.uhn.fhir.jpa.dao.tx.IHapiTransactionService; -import ca.uhn.fhir.jpa.dao.tx.NonTransactionalHapiTransactionService; +import ca.uhn.fhir.batch2.model.WorkChunkErrorEvent; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.model.api.IModelJson; import ca.uhn.fhir.util.JsonUtil; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.transaction.PlatformTransactionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Optional; import java.util.concurrent.atomic.AtomicInteger; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -48,107 +39,40 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyList; import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @SuppressWarnings({"unchecked", "rawtypes"}) @ExtendWith(MockitoExtension.class) public class WorkChunkProcessorTest { + private static final Logger ourLog = LoggerFactory.getLogger(WorkChunkProcessorTest.class); + public static final String REDUCTION_STEP_ID = "step last"; static final String INSTANCE_ID = "instanceId"; static final String JOB_DEFINITION_ID = "jobDefId"; - public static final String REDUCTION_STEP_ID = "step last"; // static internal use classes - - private enum StepType { - REDUCTION, - INTERMEDIATE, - FINAL - } - - static class TestJobParameters implements IModelJson { } - - static class StepInputData implements IModelJson { } - - static class StepOutputData implements IModelJson { } - - private static class TestDataSink extends BaseDataSink { - - private BaseDataSink myActualDataSink; - - TestDataSink(JobWorkCursor theWorkCursor) { - super(INSTANCE_ID, - theWorkCursor); - } - - public void setDataSink(BaseDataSink theSink) { - myActualDataSink = theSink; - } - - @Override - public void accept(WorkChunkData theData) { - - } - - @Override - public int getWorkChunkCount() { - return 0; - } - } - - // our test class - private class TestWorkChunkProcessor extends WorkChunkProcessor { - - public TestWorkChunkProcessor(IJobPersistence thePersistence, BatchJobSender theSender, IHapiTransactionService theHapiTransactionService) { - super(thePersistence, theSender, theHapiTransactionService); - } - - @Override - protected BaseDataSink getDataSink( - JobWorkCursor theCursor, - JobDefinition theJobDefinition, - String theInstanceId - ) { - // cause we don't want to test the actual DataSink class here! - myDataSink.setDataSink(super.getDataSink(theCursor, theJobDefinition, theInstanceId)); - return (BaseDataSink) myDataSink; - } - } - - // general mocks - - private TestDataSink myDataSink; - // step worker mocks private final IJobStepWorker myNonReductionStep = mock(IJobStepWorker.class); - private final IReductionStepWorker myReductionStep = mock(IReductionStepWorker.class); - private final ILastJobStepWorker myLastStep = mock(ILastJobStepWorker.class); - + private TestDataSink myDataSink; // class specific mocks @Mock private IJobPersistence myJobPersistence; - @Mock private BatchJobSender myJobSender; - private IHapiTransactionService myMockTransactionManager = new NonTransactionalHapiTransactionService(); - + // general mocks private TestWorkChunkProcessor myExecutorSvc; @BeforeEach public void init() { - myExecutorSvc = new TestWorkChunkProcessor(myJobPersistence, myJobSender, myMockTransactionManager); + myExecutorSvc = new TestWorkChunkProcessor(myJobPersistence, myJobSender); } private JobDefinitionStep mockOutWorkCursor( @@ -188,7 +112,6 @@ private JobDefinitionStep 0) { verify(myJobPersistence) - .incrementWorkChunkErrorCount(anyString(), eq(theRecoveredErrorsForDataSink)); + .onWorkChunkCompletion(any(WorkChunkCompletionEvent.class)); + //.workChunkErrorEvent(anyString(new WorkChunkErrorEvent(chunk.getId(), theRecoveredErrorsForDataSink))); } // nevers @@ -283,7 +207,7 @@ public void doExecute_stepWorkerThrowsJobExecutionException_marksWorkChunkAsFail runExceptionThrowingTest(new JobExecutionFailedException("Failure")); verify(myJobPersistence) - .markWorkChunkAsFailed(anyString(), anyString()); + .onWorkChunkFailed(anyString(), anyString()); } @Test @@ -315,13 +239,13 @@ public void doExecution_stepWorkerThrowsRandomExceptionForever_eventuallyMarksAs // when when(myNonReductionStep.run(any(), any())) .thenThrow(new RuntimeException(errorMsg)); - when(myJobPersistence.markWorkChunkAsErroredAndIncrementErrorCount(any(MarkWorkChunkAsErrorRequest.class))) + when(myJobPersistence.onWorkChunkError(any(WorkChunkErrorEvent.class))) .thenAnswer((p) -> { WorkChunk ec = new WorkChunk(); ec.setId(chunk.getId()); int count = errorCounter.getAndIncrement(); ec.setErrorCount(count); - return Optional.of(ec); + return count<=WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT?ec.getStatus():WorkChunkStatusEnum.FAILED; }); // test @@ -341,6 +265,7 @@ public void doExecution_stepWorkerThrowsRandomExceptionForever_eventuallyMarksAs */ processedOutcomeSuccessfully = output.isSuccessful(); } catch (JobStepFailedException ex) { + ourLog.info("Caught error:", ex); assertTrue(ex.getMessage().contains(errorMsg)); counter++; } @@ -349,12 +274,12 @@ public void doExecution_stepWorkerThrowsRandomExceptionForever_eventuallyMarksAs * we check for > MAX_CHUNK_ERROR_COUNT (+1) * we want it to run one extra time here (+1) */ - } while (processedOutcomeSuccessfully == null && counter < WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT + 2); + } while (processedOutcomeSuccessfully == null && counter <= WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT + 2); // verify assertNotNull(processedOutcomeSuccessfully); // +1 because of the > MAX_CHUNK_ERROR_COUNT check - assertEquals(WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT + 1, counter); + assertEquals(WorkChunkProcessor.MAX_CHUNK_ERROR_COUNT + 1, counter); assertFalse(processedOutcomeSuccessfully); } @@ -389,41 +314,23 @@ private void runExceptionThrowingTest(Exception theExceptionToThrow) { private void verifyNoErrors(int theRecoveredErrorCount) { if (theRecoveredErrorCount == 0) { verify(myJobPersistence, never()) - .incrementWorkChunkErrorCount(anyString(), anyInt()); + .onWorkChunkError(any()); } verify(myJobPersistence, never()) - .markWorkChunkAsFailed(anyString(), anyString()); + .onWorkChunkFailed(anyString(), anyString()); verify(myJobPersistence, never()) - .markWorkChunkAsErroredAndIncrementErrorCount(anyString(), anyString()); + .onWorkChunkError(any(WorkChunkErrorEvent.class)); } private void verifyNonReductionStep() { verify(myJobPersistence, never()) - .fetchWorkChunkSetStartTimeAndMarkInProgress(anyString()); + .onWorkChunkDequeue(anyString()); verify(myJobPersistence, never()) .markWorkChunksWithStatusAndWipeData(anyString(), anyList(), any(), any()); verify(myJobPersistence, never()) .fetchAllWorkChunksForStepStream(anyString(), anyString()); } - static JobInstance getTestJobInstance() { - JobInstance instance = JobInstance.fromInstanceId(INSTANCE_ID); - instance.setParameters(new TestJobParameters()); - - return instance; - } - - static WorkChunk createWorkChunk(String theId) { - WorkChunk chunk = new WorkChunk(); - chunk.setInstanceId(INSTANCE_ID); - chunk.setId(theId); - chunk.setStatus(StatusEnum.QUEUED); - chunk.setData(JsonUtil.serialize( - new StepInputData() - )); - return chunk; - } - @SuppressWarnings("unchecked") private JobDefinition createTestJobDefinition(boolean theWithReductionStep) { JobDefinition def = null; @@ -510,4 +417,80 @@ private JobDefinition createTestJobDefinition(boolean theWith return null; } + + private enum StepType { + REDUCTION, + INTERMEDIATE, + FINAL + } + + // our test class + private class TestWorkChunkProcessor extends WorkChunkProcessor { + + public TestWorkChunkProcessor(IJobPersistence thePersistence, BatchJobSender theSender) { + super(thePersistence, theSender); + } + + @Override + protected BaseDataSink getDataSink( + JobWorkCursor theCursor, + JobDefinition theJobDefinition, + String theInstanceId + ) { + // cause we don't want to test the actual DataSink class here! + myDataSink.setDataSink(super.getDataSink(theCursor, theJobDefinition, theInstanceId)); + return (BaseDataSink) myDataSink; + } + } + + static class TestJobParameters implements IModelJson { + } + + static class StepInputData implements IModelJson { + } + + static class StepOutputData implements IModelJson { + } + + private static class TestDataSink extends BaseDataSink { + + private BaseDataSink myActualDataSink; + + TestDataSink(JobWorkCursor theWorkCursor) { + super(INSTANCE_ID, + theWorkCursor); + } + + public void setDataSink(BaseDataSink theSink) { + myActualDataSink = theSink; + } + + @Override + public void accept(WorkChunkData theData) { + + } + + @Override + public int getWorkChunkCount() { + return 0; + } + } + + static JobInstance getTestJobInstance() { + JobInstance instance = JobInstance.fromInstanceId(INSTANCE_ID); + instance.setParameters(new TestJobParameters()); + + return instance; + } + + static WorkChunk createWorkChunk(String theId) { + WorkChunk chunk = new WorkChunk(); + chunk.setInstanceId(INSTANCE_ID); + chunk.setId(theId); + chunk.setStatus(WorkChunkStatusEnum.QUEUED); + chunk.setData(JsonUtil.serialize( + new StepInputData() + )); + return chunk; + } } diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java index 9fb02c0764cc..38b9a7ea0bec 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/maintenance/JobMaintenanceServiceImplTest.java @@ -15,13 +15,13 @@ import ca.uhn.fhir.batch2.model.JobWorkNotification; import ca.uhn.fhir.batch2.model.StatusEnum; import ca.uhn.fhir.batch2.model.WorkChunk; +import ca.uhn.fhir.batch2.model.WorkChunkStatusEnum; import ca.uhn.fhir.jpa.api.config.DaoConfig; import ca.uhn.fhir.jpa.model.sched.ISchedulerService; import ca.uhn.fhir.jpa.subscription.channel.api.IChannelProducer; import com.google.common.collect.Lists; import org.hl7.fhir.r4.model.DateTimeType; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -31,7 +31,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.messaging.Message; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -54,7 +53,6 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoMoreInteractions; @@ -74,8 +72,6 @@ public class JobMaintenanceServiceImplTest extends BaseBatch2Test { @Spy private DaoConfig myDaoConfig = new DaoConfig(); private JobMaintenanceServiceImpl mySvc; - @Captor - private ArgumentCaptor myInstanceCaptor; private JobDefinitionRegistry myJobDefinitionRegistry; @Mock private IChannelProducer myWorkChannelProducer; @@ -102,37 +98,44 @@ public void beforeEach() { @Test public void testInProgress_CalculateProgress_FirstCompleteButNoOtherStepsYetComplete() { - List chunks = new ArrayList<>(); - chunks.add(JobCoordinatorImplTest.createWorkChunk(STEP_1, null).setStatus(StatusEnum.COMPLETED)); + List chunks = List.of( + JobCoordinatorImplTest.createWorkChunk(STEP_1, null).setStatus(WorkChunkStatusEnum.COMPLETED), + JobCoordinatorImplTest.createWorkChunk(STEP_2, null).setStatus(WorkChunkStatusEnum.QUEUED) + ); + when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) + .thenReturn(chunks.iterator()); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(createInstance())); + JobInstance instance = createInstance(); + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(List.of(instance)); + when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); mySvc.runMaintenancePass(); - verify(myJobPersistence, never()).updateInstance(any()); + verify(myJobPersistence, times(1)).updateInstance(any(), any()); } @Test public void testInProgress_CalculateProgress_FirstStepComplete() { List chunks = Arrays.asList( - createWorkChunkStep1().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:01-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:01-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(createInstance())); + JobInstance instance = createInstance(); + when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); + stubUpdateInstanceCallback(instance); mySvc.runMaintenancePass(); - verify(myJobPersistence, times(1)).updateInstance(myInstanceCaptor.capture()); - JobInstance instance = myInstanceCaptor.getValue(); + verify(myJobPersistence, times(1)).updateInstance(eq(INSTANCE_ID), any()); assertEquals(0.5, instance.getProgress()); assertEquals(50, instance.getCombinedRecordsProcessed()); @@ -145,31 +148,38 @@ public void testInProgress_CalculateProgress_FirstStepComplete() { verifyNoMoreInteractions(myJobPersistence); } + private void stubUpdateInstanceCallback(JobInstance theJobInstance) { + when(myJobPersistence.updateInstance(eq(INSTANCE_ID), any())).thenAnswer(call->{ + IJobPersistence.JobInstanceUpdateCallback callback = call.getArgument(1); + return callback.doUpdate(theJobInstance); + }); + } + @Test public void testInProgress_CalculateProgress_InstanceHasErrorButNoChunksAreErrored() { // Setup List chunks = Arrays.asList( - createWorkChunkStep1().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:01-04:00")), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setErrorCount(2), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setErrorCount(2), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:01-04:00")), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setErrorCount(2), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.IN_PROGRESS).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setErrorCount(2), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); - JobInstance instance1 = createInstance(); - instance1.setErrorMessage("This is an error message"); + JobInstance instance = createInstance(); + instance.setErrorMessage("This is an error message"); when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); + stubUpdateInstanceCallback(instance); // Execute mySvc.runMaintenancePass(); // Verify - verify(myJobPersistence, times(1)).updateInstance(myInstanceCaptor.capture()); - JobInstance instance = myInstanceCaptor.getValue(); + verify(myJobPersistence, times(1)).updateInstance(eq(INSTANCE_ID), any()); assertNull(instance.getErrorMessage()); assertEquals(4, instance.getErrorCount()); @@ -184,8 +194,9 @@ public void testInProgress_CalculateProgress_InstanceHasErrorButNoChunksAreError public void testInProgress_GatedExecution_FirstStepComplete() { // Setup List chunks = Arrays.asList( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID), - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID_2) + JobCoordinatorImplTest.createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setId(CHUNK_ID + "abc"), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.QUEUED).setId(CHUNK_ID), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.QUEUED).setId(CHUNK_ID_2) ); when (myJobPersistence.canAdvanceInstanceToNextStep(any(), any())).thenReturn(true); myJobDefinitionRegistry.addJobDefinition(createJobDefinition(JobDefinition.Builder::gatedExecution)); @@ -193,20 +204,22 @@ public void testInProgress_GatedExecution_FirstStepComplete() { when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), eq(false))) .thenReturn(chunks.iterator()); - when(myJobPersistence.fetchallchunkidsforstepWithStatus(eq(INSTANCE_ID), eq(STEP_2), eq(StatusEnum.QUEUED))) - .thenReturn(chunks.stream().map(chunk -> chunk.getId()).collect(Collectors.toList())); + when(myJobPersistence.fetchAllChunkIdsForStepWithStatus(eq(INSTANCE_ID), eq(STEP_2), eq(WorkChunkStatusEnum.QUEUED))) + .thenReturn(chunks.stream().filter(c->c.getTargetStepId().equals(STEP_2)).map(WorkChunk::getId).collect(Collectors.toList())); JobInstance instance1 = createInstance(); instance1.setCurrentGatedStepId(STEP_1); when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance1)); + when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance1)); + stubUpdateInstanceCallback(instance1); // Execute mySvc.runMaintenancePass(); // Verify verify(myWorkChannelProducer, times(2)).send(myMessageCaptor.capture()); - verify(myJobPersistence, times(2)).updateInstance(myInstanceCaptor.capture()); + verify(myJobPersistence, times(2)).updateInstance(eq(INSTANCE_ID), any()); + verifyNoMoreInteractions(myJobPersistence); JobWorkNotification payload0 = myMessageCaptor.getAllValues().get(0).getPayload(); assertEquals(STEP_2, payload0.getTargetStepId()); assertEquals(CHUNK_ID, payload0.getChunkId()); @@ -222,7 +235,7 @@ public void testFailed_PurgeOldInstance() { instance.setStatus(StatusEnum.FAILED); instance.setEndTime(parseTime("2001-01-01T12:12:12Z")); when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); mySvc.runMaintenancePass(); @@ -233,33 +246,20 @@ public void testFailed_PurgeOldInstance() { @Test public void testInProgress_CalculateProgress_AllStepsComplete() { // Setup - List chunks = new ArrayList<>(); - - chunks.add( - createWorkChunkStep1().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:01:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:01-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + List chunks = List.of( + createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:01:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:01-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25),JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition(t -> t.completionHandler(myCompletionHandler))); - JobInstance instance1 = createInstance(); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); + JobInstance instance = createInstance(); + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())).thenAnswer(t->chunks.iterator()); - when(myJobPersistence.updateInstance(any())).thenReturn(true); - when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance1)); + when(myJobPersistence.fetchInstance(INSTANCE_ID)).thenReturn(Optional.of(instance)); + stubUpdateInstanceCallback(instance); // Execute @@ -267,8 +267,7 @@ public void testInProgress_CalculateProgress_AllStepsComplete() { // Verify - verify(myJobPersistence, times(2)).updateInstance(myInstanceCaptor.capture()); - JobInstance instance = myInstanceCaptor.getAllValues().get(0); + verify(myJobPersistence, times(2)).updateInstance(eq(INSTANCE_ID), any()); assertEquals(1.0, instance.getProgress()); assertEquals(StatusEnum.COMPLETED, instance.getStatus()); @@ -287,36 +286,25 @@ public void testInProgress_CalculateProgress_AllStepsComplete() { @Test public void testInProgress_CalculateProgress_OneStepFailed() { - ArrayList chunks = new ArrayList<>(); - chunks.add( - createWorkChunkStep1().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:01:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:01-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.FAILED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25).setErrorMessage("This is an error message") - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep3().setStatus(StatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) + List chunks = List.of( + createWorkChunkStep1().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:01:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:01-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.FAILED).setStartTime(parseTime("2022-02-12T14:00:02-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25).setErrorMessage("This is an error message"), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:03-04:00")).setEndTime(parseTime("2022-02-12T14:06:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep2().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:00:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25), + JobCoordinatorImplTest.createWorkChunkStep3().setStatus(WorkChunkStatusEnum.COMPLETED).setStartTime(parseTime("2022-02-12T14:01:00-04:00")).setEndTime(parseTime("2022-02-12T14:10:00-04:00")).setRecordsProcessed(25) ); myJobDefinitionRegistry.addJobDefinition(createJobDefinition()); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(createInstance())); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(createInstance())); + JobInstance instance = createInstance(); + when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance)); + when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance)); when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())) .thenAnswer(t->chunks.iterator()); + stubUpdateInstanceCallback(instance); mySvc.runMaintenancePass(); - verify(myJobPersistence, times(3)).updateInstance(myInstanceCaptor.capture()); - JobInstance instance = myInstanceCaptor.getAllValues().get(0); assertEquals(0.8333333333333334, instance.getProgress()); assertEquals(StatusEnum.FAILED, instance.getStatus()); @@ -325,79 +313,15 @@ public void testInProgress_CalculateProgress_OneStepFailed() { assertEquals(0.25, instance.getCombinedRecordsProcessedPerSecond()); assertEquals(parseTime("2022-02-12T14:10:00-04:00"), instance.getEndTime()); + // twice - once to move to FAILED, and once to purge the chunks + verify(myJobPersistence, times(2)).updateInstance(eq(INSTANCE_ID), any()); verify(myJobPersistence, times(1)).deleteChunksAndMarkInstanceAsChunksPurged(eq(INSTANCE_ID)); verifyNoMoreInteractions(myJobPersistence); } - - @Nested - public class CancellationTests { - - @Test - public void afterFirstMaintenancePass() { - // Setup - ArrayList chunks = new ArrayList<>(); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID_2) - ); - myJobDefinitionRegistry.addJobDefinition(createJobDefinition(JobDefinition.Builder::gatedExecution)); - when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())).thenAnswer(t->chunks.iterator()); - JobInstance instance1 = createInstance(); - instance1.setCurrentGatedStepId(STEP_1); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance1)); - - mySvc.runMaintenancePass(); - - // Execute - instance1.setCancelled(true); - - mySvc.runMaintenancePass(); - - // Verify - verify(myJobPersistence, times(2)).updateInstance(myInstanceCaptor.capture()); - assertEquals(StatusEnum.CANCELLED, instance1.getStatus()); - assertTrue(instance1.getErrorMessage().startsWith("Job instance cancelled")); - } - - @Test - public void afterSecondMaintenancePass() { - // Setup - ArrayList chunks = new ArrayList<>(); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID) - ); - chunks.add( - JobCoordinatorImplTest.createWorkChunkStep2().setStatus(StatusEnum.QUEUED).setId(CHUNK_ID_2) - ); - myJobDefinitionRegistry.addJobDefinition(createJobDefinition(JobDefinition.Builder::gatedExecution)); - when(myJobPersistence.fetchAllWorkChunksIterator(eq(INSTANCE_ID), anyBoolean())).thenAnswer(t->chunks.iterator()); - JobInstance instance1 = createInstance(); - instance1.setCurrentGatedStepId(STEP_1); - when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Lists.newArrayList(instance1)); - when(myJobPersistence.fetchInstance(eq(INSTANCE_ID))).thenReturn(Optional.of(instance1)); - - mySvc.runMaintenancePass(); - mySvc.runMaintenancePass(); - - // Execute - instance1.setCancelled(true); - - mySvc.runMaintenancePass(); - - // Verify - assertEquals(StatusEnum.CANCELLED, instance1.getStatus()); - assertTrue(instance1.getErrorMessage().startsWith("Job instance cancelled")); - } - - } - @Test - void triggerMaintenancePass_noneInProgress_runsMaintenace() { + void triggerMaintenancePass_noneInProgress_runsMaintenance() { when(myJobPersistence.fetchInstances(anyInt(), eq(0))).thenReturn(Collections.emptyList()); mySvc.triggerMaintenancePass(); diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/JobInstanceTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/JobInstanceTest.java new file mode 100644 index 000000000000..80732d8e4f7b --- /dev/null +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/JobInstanceTest.java @@ -0,0 +1,26 @@ +package ca.uhn.fhir.batch2.model; + +import ca.uhn.fhir.test.utilities.RandomDataHelper; +import org.apache.commons.lang3.builder.EqualsBuilder; +import org.junit.jupiter.api.Test; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.*; + +class JobInstanceTest { + + @Test + void testCopyConstructor_randomFieldsCopied_areEqual() { + // given + JobInstance instance = new JobInstance(); + RandomDataHelper.fillFieldsRandomly(instance); + + // when + JobInstance copy = new JobInstance(instance); + + // then + assertTrue(EqualsBuilder.reflectionEquals(instance, copy)); + } + +} diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/StatusEnumTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/StatusEnumTest.java index c89f7ed1c2c2..f319d240cf09 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/StatusEnumTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/StatusEnumTest.java @@ -3,19 +3,22 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.EnumSource; +import static org.hamcrest.CoreMatchers.not; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.hasItem; import static org.junit.jupiter.api.Assertions.assertEquals; class StatusEnumTest { @Test public void testEndedStatuses() { - assertThat(StatusEnum.getEndedStatuses(), containsInAnyOrder(StatusEnum.COMPLETED, StatusEnum.FAILED, StatusEnum.CANCELLED, StatusEnum.ERRORED)); + assertThat(StatusEnum.getEndedStatuses(), containsInAnyOrder(StatusEnum.COMPLETED, StatusEnum.FAILED, StatusEnum.CANCELLED)); } @Test public void testNotEndedStatuses() { - assertThat(StatusEnum.getNotEndedStatuses(), containsInAnyOrder(StatusEnum.QUEUED, StatusEnum.IN_PROGRESS, StatusEnum.FINALIZE)); + assertThat(StatusEnum.getNotEndedStatuses(), containsInAnyOrder(StatusEnum.QUEUED, StatusEnum.IN_PROGRESS, StatusEnum.ERRORED, StatusEnum.FINALIZE)); } @ParameterizedTest @@ -36,7 +39,7 @@ public void testNotEndedStatuses() { "COMPLETED, QUEUED, false", "COMPLETED, IN_PROGRESS, false", - "COMPLETED, COMPLETED, true", + "COMPLETED, COMPLETED, false", "COMPLETED, CANCELLED, false", "COMPLETED, ERRORED, false", "COMPLETED, FAILED, false", @@ -44,7 +47,7 @@ public void testNotEndedStatuses() { "CANCELLED, QUEUED, false", "CANCELLED, IN_PROGRESS, false", "CANCELLED, COMPLETED, false", - "CANCELLED, CANCELLED, true", + "CANCELLED, CANCELLED, false", "CANCELLED, ERRORED, false", "CANCELLED, FAILED, false", @@ -69,6 +72,19 @@ public void testNotEndedStatuses() { }) public void testStateTransition(StatusEnum origStatus, StatusEnum newStatus, boolean expected) { assertEquals(expected, StatusEnum.isLegalStateTransition(origStatus, newStatus)); + if (expected) { + assertThat(StatusEnum.ourFromStates.get(newStatus), hasItem(origStatus)); + assertThat(StatusEnum.ourToStates.get(origStatus), hasItem(newStatus)); + } else { + assertThat(StatusEnum.ourFromStates.get(newStatus), not(hasItem(origStatus))); + assertThat(StatusEnum.ourToStates.get(origStatus), not(hasItem(newStatus))); + } + } + + @ParameterizedTest + @EnumSource(StatusEnum.class) + public void testCancellableStates(StatusEnum theState) { + assertEquals(StatusEnum.ourFromStates.get(StatusEnum.CANCELLED).contains(theState), theState.isCancellable()); } @Test diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnumTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnumTest.java new file mode 100644 index 000000000000..6800ffe4dd43 --- /dev/null +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/model/WorkChunkStatusEnumTest.java @@ -0,0 +1,37 @@ +package ca.uhn.fhir.batch2.model; + +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.EnumSource; + +import java.util.Arrays; + +import static org.hamcrest.CoreMatchers.hasItem; +import static org.hamcrest.CoreMatchers.not; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class WorkChunkStatusEnumTest { + @ParameterizedTest + @EnumSource(WorkChunkStatusEnum.class) + void allStatesExceptCOMPLETEDareIncomplete(WorkChunkStatusEnum theEnum) { + if (theEnum == WorkChunkStatusEnum.COMPLETED) { + assertFalse(theEnum.isIncomplete()); + } else { + assertTrue(theEnum.isIncomplete()); + } + } + + @ParameterizedTest + @EnumSource(WorkChunkStatusEnum.class) + void allowedPriorStates_matchesNextStates(WorkChunkStatusEnum theEnum) { + Arrays.stream(WorkChunkStatusEnum.values()).forEach(nextPrior->{ + if (nextPrior.getNextStates().contains(theEnum)) { + assertThat("is prior", theEnum.getPriorStates(), hasItem(nextPrior)); + } else { + assertThat("is not prior", theEnum.getPriorStates(), not(hasItem(nextPrior))); + } + }); + } +} diff --git a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdaterTest.java b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdaterTest.java index 3714931b7972..b0b57ce688b1 100644 --- a/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdaterTest.java +++ b/hapi-fhir-storage-batch2/src/test/java/ca/uhn/fhir/batch2/progress/JobInstanceStatusUpdaterTest.java @@ -2,7 +2,6 @@ import ca.uhn.fhir.batch2.api.IJobCompletionHandler; import ca.uhn.fhir.batch2.api.IJobInstance; -import ca.uhn.fhir.batch2.api.IJobPersistence; import ca.uhn.fhir.batch2.api.JobCompletionDetails; import ca.uhn.fhir.batch2.coordinator.JobDefinitionRegistry; import ca.uhn.fhir.batch2.model.JobDefinition; @@ -17,7 +16,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.util.Optional; import java.util.concurrent.atomic.AtomicReference; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -33,8 +31,6 @@ class JobInstanceStatusUpdaterTest { private static final int TEST_ERROR_COUNT = 729; private final JobInstance myQueuedInstance = new JobInstance().setStatus(StatusEnum.QUEUED); - @Mock - IJobPersistence myJobPersistence; @Mock private JobDefinition myJobDefinition; @Mock @@ -91,8 +87,6 @@ private void assertCompleteCallbackCalled() { private void setupCompleteCallback() { myDetails = new AtomicReference<>(); - when(myJobPersistence.fetchInstance(TEST_INSTANCE_ID)).thenReturn(Optional.of(myQueuedInstance)); - when(myJobPersistence.updateInstance(myInstance)).thenReturn(true); IJobCompletionHandler completionHandler = details -> myDetails.set(details); when(myJobDefinition.getCompletionHandler()).thenReturn(completionHandler); when(myJobDefinition.getParametersType()).thenReturn(TestParameters.class); @@ -102,8 +96,6 @@ private void setupCompleteCallback() { public void testErrorHandler_ERROR() { // setup myDetails = new AtomicReference<>(); - when(myJobPersistence.fetchInstance(TEST_INSTANCE_ID)).thenReturn(Optional.of(myQueuedInstance)); - when(myJobPersistence.updateInstance(myInstance)).thenReturn(true); // execute mySvc.updateInstanceStatus(myInstance, StatusEnum.ERRORED); @@ -145,8 +137,6 @@ private void setupErrorCallback() { myDetails = new AtomicReference<>(); // setup - when(myJobPersistence.fetchInstance(TEST_INSTANCE_ID)).thenReturn(Optional.of(myQueuedInstance)); - when(myJobPersistence.updateInstance(myInstance)).thenReturn(true); IJobCompletionHandler errorHandler = details -> myDetails.set(details); when(myJobDefinition.getErrorHandler()).thenReturn(errorHandler); when(myJobDefinition.getParametersType()).thenReturn(TestParameters.class); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java index b429cdf2729e..4d6bcc3e7920 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/model/BulkExportParameters.java @@ -82,6 +82,9 @@ public class BulkExportParameters extends Batch2BaseJobParameters { * The request which originated the request. */ private String myOriginalRequestUrl; + private String myExportIdentifier; + + public boolean isExpandMdm() { return myExpandMdm; @@ -99,6 +102,13 @@ public List getResourceTypes() { return myResourceTypes; } + public void setExportIdentifier(String theExportIdentifier) { + myExportIdentifier = theExportIdentifier; + } + public String getExportIdentifier() { + return myExportIdentifier; + } + public void setResourceTypes(List theResourceTypes) { myResourceTypes = theResourceTypes; } diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IIdHelperService.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IIdHelperService.java index 195dbb1debe5..f01299d9887d 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IIdHelperService.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/api/svc/IIdHelperService.java @@ -133,7 +133,7 @@ public interface IIdHelperService { Optional translatePidIdToForcedIdWithCache(T theResourcePersistentId); - PersistentIdToForcedIdMap translatePidsToForcedIds(Set theResourceIds); + PersistentIdToForcedIdMap translatePidsToForcedIds(Set theResourceIds); /** * Pre-cache a PID-to-Resource-ID mapping for later retrieval by {@link #translatePidsToForcedIds(Set)} and related methods diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/api/IBinaryStorageSvc.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/api/IBinaryStorageSvc.java index 48173309c17f..42af18818192 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/api/IBinaryStorageSvc.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/api/IBinaryStorageSvc.java @@ -37,6 +37,16 @@ public interface IBinaryStorageSvc { long getMaximumBinarySize(); /** + * Given a blob ID, return true if it is valid for the underlying storage mechanism, false otherwise. + * + * @param theNewBlobId the blob ID to validate + * @return true if the blob ID is valid, false otherwise. + */ + default boolean isValidBlobId(String theNewBlobId) { + return true;//default method here as we don't want to break existing implementations + } + + /** * Sets the maximum number of bytes that can be stored in a single binary * file by this service. The default is {@link Long#MAX_VALUE} * diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/interceptor/BinaryStorageInterceptor.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/interceptor/BinaryStorageInterceptor.java index cfe460f941fd..6005706b5918 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/interceptor/BinaryStorageInterceptor.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/interceptor/BinaryStorageInterceptor.java @@ -25,6 +25,8 @@ import ca.uhn.fhir.context.FhirContext; import ca.uhn.fhir.i18n.Msg; import ca.uhn.fhir.interceptor.api.Hook; +import ca.uhn.fhir.interceptor.api.HookParams; +import ca.uhn.fhir.interceptor.api.IInterceptorBroadcaster; import ca.uhn.fhir.interceptor.api.Interceptor; import ca.uhn.fhir.interceptor.api.Pointcut; import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc; @@ -32,18 +34,27 @@ import ca.uhn.fhir.jpa.binary.api.StoredDetails; import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider; import ca.uhn.fhir.jpa.model.util.JpaConstants; +import ca.uhn.fhir.rest.api.server.IPreResourceAccessDetails; import ca.uhn.fhir.rest.api.server.IPreResourceShowDetails; +import ca.uhn.fhir.rest.api.server.RequestDetails; +import ca.uhn.fhir.rest.api.server.SimplePreResourceAccessDetails; import ca.uhn.fhir.rest.api.server.storage.TransactionDetails; +import ca.uhn.fhir.rest.server.exceptions.InternalErrorException; import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException; +import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException; +import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails; +import ca.uhn.fhir.rest.server.util.CompositeInterceptorBroadcaster; import ca.uhn.fhir.util.HapiExtensions; import ca.uhn.fhir.util.IModelVisitor2; import org.apache.commons.io.FileUtils; import org.hl7.fhir.instance.model.api.IBase; +import org.hl7.fhir.instance.model.api.IBaseBinary; import org.hl7.fhir.instance.model.api.IBaseHasExtensions; import org.hl7.fhir.instance.model.api.IBaseResource; import org.hl7.fhir.instance.model.api.IIdType; import org.hl7.fhir.instance.model.api.IPrimitiveType; import org.hl7.fhir.r4.model.IdType; +import org.hl7.fhir.r4.model.Request; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -72,6 +83,9 @@ public class BinaryStorageInterceptor> { private final FhirContext myCtx; @Autowired private BinaryAccessProvider myBinaryAccessProvider; + + @Autowired + private IInterceptorBroadcaster myInterceptorBroadcaster; private Class myBinaryType; private String myDeferredListKey; private long myAutoInflateBinariesMaximumBytes = 10 * FileUtils.ONE_MB; @@ -124,14 +138,14 @@ public void expungeResource(AtomicInteger theCounter, IBaseResource theResource) } @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED) - public void extractLargeBinariesBeforeCreate(TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut) throws IOException { - extractLargeBinaries(theTransactionDetails, theResource, thePointcut); + public void extractLargeBinariesBeforeCreate(RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut) throws IOException { + extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut); } @Hook(Pointcut.STORAGE_PRESTORAGE_RESOURCE_UPDATED) - public void extractLargeBinariesBeforeUpdate(TransactionDetails theTransactionDetails, IBaseResource thePreviousResource, IBaseResource theResource, Pointcut thePointcut) throws IOException { + public void extractLargeBinariesBeforeUpdate(RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, IBaseResource thePreviousResource, IBaseResource theResource, Pointcut thePointcut) throws IOException { blockIllegalExternalBinaryIds(thePreviousResource, theResource); - extractLargeBinaries(theTransactionDetails, theResource, thePointcut); + extractLargeBinaries(theRequestDetails, theTransactionDetails, theResource, thePointcut); } /** @@ -181,7 +195,7 @@ private void blockIllegalExternalBinaryIds(IBaseResource thePreviousResource, IB } - private void extractLargeBinaries(TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut) throws IOException { + private void extractLargeBinaries(RequestDetails theRequestDetails, TransactionDetails theTransactionDetails, IBaseResource theResource, Pointcut thePointcut) throws IOException { IIdType resourceId = theResource.getIdElement(); if (!resourceId.hasResourceType() && resourceId.hasIdPart()) { @@ -207,9 +221,18 @@ private void extractLargeBinaries(TransactionDetails theTransactionDetails, IBas } else { assert thePointcut == Pointcut.STORAGE_PRESTORAGE_RESOURCE_CREATED : thePointcut.name(); newBlobId = myBinaryStorageSvc.newBlobId(); - List deferredBinaryTargets = getOrCreateDeferredBinaryStorageMap(theTransactionDetails); - DeferredBinaryTarget newDeferredBinaryTarget = new DeferredBinaryTarget(newBlobId, nextTarget, data); - deferredBinaryTargets.add(newDeferredBinaryTarget); + + String prefix = invokeAssignBlobPrefix(theRequestDetails, theResource); + if (isNotBlank(prefix)) { + newBlobId = prefix + newBlobId; + } + if (myBinaryStorageSvc.isValidBlobId(newBlobId)) { + List deferredBinaryTargets = getOrCreateDeferredBinaryStorageMap(theTransactionDetails); + DeferredBinaryTarget newDeferredBinaryTarget = new DeferredBinaryTarget(newBlobId, nextTarget, data); + deferredBinaryTargets.add(newDeferredBinaryTarget); + } else { + throw new InternalErrorException(Msg.code(2341) + "Invalid blob ID for backing storage service.[blobId=" + newBlobId + ",service=" + myBinaryStorageSvc.getClass().getName() +"]"); + } } myBinaryAccessProvider.replaceDataWithExtension(nextTarget, newBlobId); @@ -218,6 +241,21 @@ private void extractLargeBinaries(TransactionDetails theTransactionDetails, IBas } } + /** + * This invokes the {@link Pointcut#STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX} hook and returns the prefix to use for the blob ID, or null if there are no implementers. + * @return A string, which will be used to prefix the blob ID. May be null. + */ + private String invokeAssignBlobPrefix(RequestDetails theRequest, IBaseResource theResource) { + if (CompositeInterceptorBroadcaster.hasHooks(Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, myInterceptorBroadcaster, theRequest)) { + HookParams params = new HookParams() + .add(RequestDetails.class, theRequest) + .add(IBaseResource.class, theResource); + return (String) CompositeInterceptorBroadcaster.doCallHooksAndReturnObject(myInterceptorBroadcaster, theRequest, Pointcut.STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX, params); + } else { + return null; + } + } + @Nonnull private List getOrCreateDeferredBinaryStorageMap(TransactionDetails theTransactionDetails) { return theTransactionDetails.getOrCreateUserData(getDeferredListKey(), () -> new ArrayList<>()); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/BaseBinaryStorageSvcImpl.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/BaseBinaryStorageSvcImpl.java index 87ec98d72615..436b0e2d3144 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/BaseBinaryStorageSvcImpl.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/BaseBinaryStorageSvcImpl.java @@ -95,6 +95,14 @@ public String newBlobId() { return b.toString(); } + /** + * Default implementation is to return true for any Blob ID. + */ + @Override + public boolean isValidBlobId(String theNewBlobId) { + return true; + } + @Override public boolean shouldStoreBlob(long theSize, IIdType theResourceId, String theContentType) { return theSize >= getMinimumBinarySize(); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/NullBinaryStorageSvcImpl.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/NullBinaryStorageSvcImpl.java index f948a962053d..b46c42529866 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/NullBinaryStorageSvcImpl.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binary/svc/NullBinaryStorageSvcImpl.java @@ -37,6 +37,11 @@ public long getMaximumBinarySize() { return 0; } + @Override + public boolean isValidBlobId(String theNewBlobId) { + return true; + } + @Override public void setMaximumBinarySize(long theMaximumBinarySize) { // ignore diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binstore/FilesystemBinaryStorageSvcImpl.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binstore/FilesystemBinaryStorageSvcImpl.java index 25daa4dd4c24..2e0a920822ae 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binstore/FilesystemBinaryStorageSvcImpl.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/binstore/FilesystemBinaryStorageSvcImpl.java @@ -33,6 +33,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.io.input.CountingInputStream; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.Validate; import org.hl7.fhir.instance.model.api.IIdType; import org.slf4j.Logger; @@ -51,6 +52,8 @@ import java.io.OutputStream; import java.io.Reader; import java.util.Date; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class FilesystemBinaryStorageSvcImpl extends BaseBinaryStorageSvcImpl { @@ -76,6 +79,14 @@ private void createBasePathDirectory() { mkdir(myBasePath); } + /** + * This implementation prevents: \ / | . + */ + @Override + public boolean isValidBlobId(String theNewBlobId) { + return !StringUtils.containsAny(theNewBlobId, '\\', '/', '|', '.'); + + } @Override public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, String theContentType, InputStream theInputStream) throws IOException { String id = super.provideIdForNewBlob(theBlobIdOrNull); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/provider/BulkDataExportProvider.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/provider/BulkDataExportProvider.java index 3486de4e8731..2ae132308374 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/provider/BulkDataExportProvider.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/bulk/export/provider/BulkDataExportProvider.java @@ -121,12 +121,13 @@ public void export( @OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType theSince, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypeFilter, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_POST_FETCH_FILTER_URL, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypePostFetchFilterUrl, + @OperationParam(name = JpaConstants.PARAM_EXPORT_IDENTIFIER, min = 0, max = 1, typeName = "string") IPrimitiveType theExportId, ServletRequestDetails theRequestDetails ) { // JPA export provider validatePreferAsyncHeader(theRequestDetails, JpaConstants.OPERATION_EXPORT); - BulkDataExportOptions bulkDataExportOptions = buildSystemBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theTypePostFetchFilterUrl); + BulkDataExportOptions bulkDataExportOptions = buildSystemBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportId, theTypePostFetchFilterUrl); startJob(theRequestDetails, bulkDataExportOptions); } @@ -208,6 +209,7 @@ public void groupExport( @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypeFilter, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_POST_FETCH_FILTER_URL, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypePostFetchFilterUrl, @OperationParam(name = JpaConstants.PARAM_EXPORT_MDM, min = 0, max = 1, typeName = "boolean") IPrimitiveType theMdm, + @OperationParam(name = JpaConstants.PARAM_EXPORT_IDENTIFIER, min = 0, max = 1, typeName = "string") IPrimitiveType theExportIdentifier, ServletRequestDetails theRequestDetails ) { ourLog.debug("Received Group Bulk Export Request for Group {}", theIdParam); @@ -218,7 +220,7 @@ public void groupExport( validatePreferAsyncHeader(theRequestDetails, JpaConstants.OPERATION_EXPORT); - BulkDataExportOptions bulkDataExportOptions = buildGroupBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theIdParam, theMdm, theTypePostFetchFilterUrl); + BulkDataExportOptions bulkDataExportOptions = buildGroupBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theIdParam, theMdm, theExportIdentifier, theTypePostFetchFilterUrl); if (isNotEmpty(bulkDataExportOptions.getResourceTypes())) { validateResourceTypesAllContainPatientSearchParams(bulkDataExportOptions.getResourceTypes()); @@ -262,10 +264,11 @@ public void patientExport( @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypeFilter, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_POST_FETCH_FILTER_URL, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypePostFetchFilterUrl, @OperationParam(name = JpaConstants.PARAM_EXPORT_PATIENT, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> thePatient, + @OperationParam(name = JpaConstants.PARAM_EXPORT_IDENTIFIER, min = 0, max = 1, typeName = "string") IPrimitiveType theExportIdentifier, ServletRequestDetails theRequestDetails ) { validatePreferAsyncHeader(theRequestDetails, JpaConstants.OPERATION_EXPORT); - BulkDataExportOptions bulkDataExportOptions = buildPatientBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, thePatient, theTypePostFetchFilterUrl); + BulkDataExportOptions bulkDataExportOptions = buildPatientBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportIdentifier, thePatient, theTypePostFetchFilterUrl); validateResourceTypesAllContainPatientSearchParams(bulkDataExportOptions.getResourceTypes()); startJob(theRequestDetails, bulkDataExportOptions); @@ -282,10 +285,11 @@ public void patientInstanceExport( @OperationParam(name = JpaConstants.PARAM_EXPORT_SINCE, min = 0, max = 1, typeName = "instant") IPrimitiveType theSince, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_FILTER, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypeFilter, @OperationParam(name = JpaConstants.PARAM_EXPORT_TYPE_POST_FETCH_FILTER_URL, min = 0, max = OperationParam.MAX_UNLIMITED, typeName = "string") List> theTypePostFetchFilterUrl, + @OperationParam(name = JpaConstants.PARAM_EXPORT_IDENTIFIER, min = 0, max = 1, typeName = "string") IPrimitiveType theExportIdentifier, ServletRequestDetails theRequestDetails ) { validatePreferAsyncHeader(theRequestDetails, JpaConstants.OPERATION_EXPORT); - BulkDataExportOptions bulkDataExportOptions = buildPatientBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theIdParam, theTypePostFetchFilterUrl); + BulkDataExportOptions bulkDataExportOptions = buildPatientBulkExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportIdentifier, theIdParam, theTypePostFetchFilterUrl); validateResourceTypesAllContainPatientSearchParams(bulkDataExportOptions.getResourceTypes()); startJob(theRequestDetails, bulkDataExportOptions); @@ -418,12 +422,12 @@ private String getTransitionTimeOfJobInfo(Batch2JobInfo theInfo) { } } - private BulkDataExportOptions buildSystemBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, List> theTypePostFetchFilterUrl) { - return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.SYSTEM, theTypePostFetchFilterUrl); + private BulkDataExportOptions buildSystemBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IPrimitiveType theExportId, List> theTypePostFetchFilterUrl) { + return buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportId, BulkDataExportOptions.ExportStyle.SYSTEM, theTypePostFetchFilterUrl); } - private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IIdType theGroupId, IPrimitiveType theExpandMdm, List> theTypePostFetchFilterUrl) { - BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.GROUP, theTypePostFetchFilterUrl); + private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IIdType theGroupId, IPrimitiveType theExpandMdm, IPrimitiveType theExportId, List> theTypePostFetchFilterUrl) { + BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportId, BulkDataExportOptions.ExportStyle.GROUP, theTypePostFetchFilterUrl); bulkDataExportOptions.setGroupId(theGroupId); boolean mdm = false; @@ -435,26 +439,26 @@ private BulkDataExportOptions buildGroupBulkExportOptions(IPrimitiveType return bulkDataExportOptions; } - private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, List> thePatientIds, List> theTypePostFetchFilterUrl) { + private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IPrimitiveType theExportIdentifier, List> thePatientIds, List> theTypePostFetchFilterUrl) { IPrimitiveType type = theType; if (type == null) { // Type is optional, but the job requires it type = new StringDt("Patient"); } - BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, type, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.PATIENT, theTypePostFetchFilterUrl); + BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, type, theSince, theTypeFilter, theExportIdentifier, BulkDataExportOptions.ExportStyle.PATIENT, theTypePostFetchFilterUrl); if (thePatientIds != null) { bulkDataExportOptions.setPatientIds(thePatientIds.stream().map((pid) -> new IdType(pid.getValueAsString())).collect(Collectors.toSet())); } return bulkDataExportOptions; } - private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IIdType thePatientId, List> theTypePostFetchFilterUrl) { - BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, BulkDataExportOptions.ExportStyle.PATIENT, theTypePostFetchFilterUrl); + private BulkDataExportOptions buildPatientBulkExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IPrimitiveType theExportIdentifier, IIdType thePatientId, List> theTypePostFetchFilterUrl) { + BulkDataExportOptions bulkDataExportOptions = buildBulkDataExportOptions(theOutputFormat, theType, theSince, theTypeFilter, theExportIdentifier, BulkDataExportOptions.ExportStyle.PATIENT, theTypePostFetchFilterUrl); bulkDataExportOptions.setPatientIds(Collections.singleton(thePatientId)); return bulkDataExportOptions; } - private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, BulkDataExportOptions.ExportStyle theExportStyle, List> theTypePostFetchFilterUrl) { + private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType theOutputFormat, IPrimitiveType theType, IPrimitiveType theSince, List> theTypeFilter, IPrimitiveType theExportIdentifier, BulkDataExportOptions.ExportStyle theExportStyle, List> theTypePostFetchFilterUrl) { String outputFormat = theOutputFormat != null ? theOutputFormat.getValueAsString() : Constants.CT_FHIR_NDJSON; Set resourceTypes = null; @@ -466,6 +470,10 @@ private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType if (theSince != null) { since = theSince.getValue(); } + String exportIdentifier = null; + if (theExportIdentifier != null) { + exportIdentifier = theExportIdentifier.getValueAsString(); + } Set typeFilters = splitTypeFilters(theTypeFilter); Set typePostFetchFilterUrls = splitTypeFilters(theTypePostFetchFilterUrl); @@ -474,6 +482,7 @@ private BulkDataExportOptions buildBulkDataExportOptions(IPrimitiveType bulkDataExportOptions.setFilters(typeFilters); bulkDataExportOptions.setPostFetchFilterUrls(typePostFetchFilterUrls); bulkDataExportOptions.setExportStyle(theExportStyle); + bulkDataExportOptions.setExportIdentifier(exportIdentifier); bulkDataExportOptions.setSince(since); bulkDataExportOptions.setResourceTypes(resourceTypes); bulkDataExportOptions.setOutputFormat(outputFormat); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java index 1ac6a1c6bedb..c3b36e72b40d 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/HapiTransactionService.java @@ -85,6 +85,11 @@ public IExecutionBuilder withRequest(@Nullable RequestDetails theRequestDetails) return new ExecutionBuilder(theRequestDetails); } + @Override + public IExecutionBuilder withSystemRequest() { + return new ExecutionBuilder(null); + } + /** * @deprecated Use {@link #withRequest(RequestDetails)} with fluent call instead diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/IHapiTransactionService.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/IHapiTransactionService.java index 2cc4b96b188f..6824ca6a2d10 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/IHapiTransactionService.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/IHapiTransactionService.java @@ -49,12 +49,19 @@ public interface IHapiTransactionService { */ IExecutionBuilder withRequest(@Nullable RequestDetails theRequestDetails); + /** + * Fluent builder for internal system requests with no external + * requestdetails associated + */ + IExecutionBuilder withSystemRequest(); + /** * @deprecated It is highly recommended to use {@link #withRequest(RequestDetails)} instead of this method, for increased visibility. */ @Deprecated T withRequest(@Nullable RequestDetails theRequestDetails, @Nullable TransactionDetails theTransactionDetails, @Nonnull Propagation thePropagation, @Nonnull Isolation theIsolation, @Nonnull ICallable theCallback); + interface IExecutionBuilder { IExecutionBuilder withIsolation(Isolation theIsolation); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/NonTransactionalHapiTransactionService.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/NonTransactionalHapiTransactionService.java index 42df6d1ef602..47af49ce8ec1 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/NonTransactionalHapiTransactionService.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/dao/tx/NonTransactionalHapiTransactionService.java @@ -20,6 +20,7 @@ * #L% */ +import org.springframework.transaction.support.SimpleTransactionStatus; import org.springframework.transaction.support.TransactionCallback; import javax.annotation.Nullable; @@ -34,6 +35,6 @@ public class NonTransactionalHapiTransactionService extends HapiTransactionServi @Nullable @Override protected T doExecute(ExecutionBuilder theExecutionBuilder, TransactionCallback theCallback) { - return theCallback.doInTransaction(null); + return theCallback.doInTransaction(new SimpleTransactionStatus()); } } diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/subscription/channel/impl/RetryingMessageHandlerWrapper.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/subscription/channel/impl/RetryingMessageHandlerWrapper.java index 98ff218d5f23..6758c3720c02 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/subscription/channel/impl/RetryingMessageHandlerWrapper.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/subscription/channel/impl/RetryingMessageHandlerWrapper.java @@ -34,6 +34,8 @@ import org.springframework.retry.listener.RetryListenerSupport; import org.springframework.retry.policy.TimeoutRetryPolicy; import org.springframework.retry.support.RetryTemplate; +import org.springframework.transaction.CannotCreateTransactionException; +import org.springframework.transaction.TransactionException; import javax.annotation.Nonnull; @@ -67,6 +69,15 @@ public void onError(RetryContext theContext, RetryCallb if (theThrowable instanceof BaseUnrecoverableRuntimeException) { theContext.setExhaustedOnly(); } + if (theThrowable instanceof CannotCreateTransactionException) { + /* + * This exception means that we can't open a transaction, which + * means the EntityManager is closed. This can happen if we are shutting + * down while there is still a message in the queue - No sense + * retrying indefinitely in that case + */ + theContext.setExhaustedOnly(); + } } }; retryTemplate.setListeners(new RetryListener[]{retryListener}); diff --git a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/util/BulkExportUtils.java b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/util/BulkExportUtils.java index fd2d8b6544e9..99c8e584eee5 100644 --- a/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/util/BulkExportUtils.java +++ b/hapi-fhir-storage/src/main/java/ca/uhn/fhir/jpa/util/BulkExportUtils.java @@ -54,6 +54,7 @@ public static BulkExportParameters createBulkExportJobParametersFromExportOption } parameters.setExpandMdm(theOptions.isExpandMdm()); parameters.setUseExistingJobsFirst(true); + parameters.setExportIdentifier(theOptions.getExportIdentifier()); return parameters; } diff --git a/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/RandomDataHelper.java b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/RandomDataHelper.java new file mode 100644 index 000000000000..cdbe1ec8b4b6 --- /dev/null +++ b/hapi-fhir-test-utilities/src/main/java/ca/uhn/fhir/test/utilities/RandomDataHelper.java @@ -0,0 +1,76 @@ +package ca.uhn.fhir.test.utilities; + +/*- + * #%L + * HAPI FHIR Test Utilities + * %% + * Copyright (C) 2014 - 2023 Smile CDR, Inc. + * %% + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * #L% + */ + +import org.apache.commons.lang3.Validate; +import org.springframework.util.ReflectionUtils; + +import java.lang.reflect.Modifier; +import java.util.Date; +import java.util.Random; +import java.util.UUID; + +public class RandomDataHelper { + public static void fillFieldsRandomly(Object theTarget) { + new RandomDataHelper().fillFields(theTarget); + } + + public void fillFields(Object theTarget) { + ReflectionUtils.doWithFields(theTarget.getClass(), field->{ + Class fieldType = field.getType(); + if (!Modifier.isFinal(field.getModifiers())) { + ReflectionUtils.makeAccessible(field); + Object value = generateRandomValue(fieldType); + field.set(theTarget, value); + } + }); + } + + + public Object generateRandomValue(Class fieldType) { + Random random = new Random(); + Object result = null; + if (fieldType.equals(String.class)) { + result = UUID.randomUUID().toString(); + } else if (fieldType.equals(UUID.class)) { + result = UUID.randomUUID(); + } else if (Date.class.isAssignableFrom(fieldType)) { + result = new Date(System.currentTimeMillis() - random.nextInt(100000000)); + } else if (fieldType.equals(Integer.TYPE)) { + result = random.nextInt(); + } else if (fieldType.equals(Long.TYPE)) { + result = random.nextInt(); + } else if (fieldType.equals(Long.class)) { + result = random.nextLong(); + } else if (fieldType.equals(Double.class) || fieldType.equals(Double.TYPE)) { + result = random.nextDouble(); + } else if (Number.class.isAssignableFrom(fieldType)) { + result = random.nextInt(Byte.MAX_VALUE) + 1; + } else if (Enum.class.isAssignableFrom(fieldType)) { + Object[] enumValues = fieldType.getEnumConstants(); + result = enumValues[random.nextInt(enumValues.length)]; + } else if (fieldType.equals(Boolean.TYPE) || fieldType.equals(Boolean.class)) { + result = random.nextBoolean(); + } + Validate.notNull(result, "Does not support type %s", fieldType); + return result; + } +} diff --git a/hapi-fhir-test-utilities/src/test/java/ca/uhn/fhir/test/utilities/RandomDataHelperTest.java b/hapi-fhir-test-utilities/src/test/java/ca/uhn/fhir/test/utilities/RandomDataHelperTest.java new file mode 100644 index 000000000000..85d20a3d15d0 --- /dev/null +++ b/hapi-fhir-test-utilities/src/test/java/ca/uhn/fhir/test/utilities/RandomDataHelperTest.java @@ -0,0 +1,40 @@ +package ca.uhn.fhir.test.utilities; + +import ca.uhn.fhir.model.api.TemporalPrecisionEnum; +import org.junit.jupiter.api.Test; + +import java.util.Date; +import java.util.UUID; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.blankOrNullString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; + +class RandomDataHelperTest { + + static class TestClass { + String myString; + int myInt; + Long myBoxedLong; + Date myDate; + UUID myUUID; + TemporalPrecisionEnum myEnum; + } + + @Test + void fillFieldsRandomly() { + TestClass object = new TestClass(); + + RandomDataHelper.fillFieldsRandomly(object); + + assertThat(object.myString, not(blankOrNullString())); + assertThat(object.myInt, not(equalTo(0))); + assertThat(object.myBoxedLong, notNullValue()); + assertThat(object.myDate, notNullValue()); + assertThat(object.myUUID, notNullValue()); + assertThat(object.myEnum, notNullValue()); + } + +} diff --git a/pom.xml b/pom.xml index d6ac6d6be0de..922c5314ee1f 100644 --- a/pom.xml +++ b/pom.xml @@ -97,6 +97,7 @@ hapi-fhir-serviceloaders hapi-fhir-storage hapi-fhir-storage-batch2 + hapi-fhir-storage-batch2-test-utilities hapi-fhir-storage-batch2-jobs hapi-fhir-storage-cr hapi-fhir-storage-mdm