Make WorkChunk handling transactional at state transitions. (#4621)

- use a separate enum for the states - chunks have different transitions than instances.
- use transactional update events for work-chunk state transitions
- introduce spec-test to define behaviour of batch2 storage
- replace synchronized facade with simpler ProxyUtil handler.
- change job cancellation to db update query

hapi-fhir-base/src/main/java/ca/uhn/fhir/util/ProxyUtil.java

@@ -0,0 +1,47 @@
+package ca.uhn.fhir.util;
+
+import org.apache.commons.lang3.Validate;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class ProxyUtil {
+ private ProxyUtil() {}
+
+ /**
+ * Wrap theInstance in a Proxy that synchronizes every method.
+ *
+ * @param theClass the target interface
+ * @param theInstance the instance to wrap
+ * @return a Proxy implementing theClass interface that syncronizes every call on theInstance
+ * @param <T> the interface type
+ */
+ public static <T> T synchronizedProxy(Class<T> theClass, T theInstance) {
+ Validate.isTrue(theClass.isInterface(), "%s is not an interface", theClass);
+ InvocationHandler handler = new SynchronizedHandler(theInstance);
+ Object object = Proxy.newProxyInstance(theClass.getClassLoader(), new Class<?>[] { theClass }, handler);
+ return theClass.cast(object);
+ }
+
+ /**
+ * Simple handler that first synchronizes on the delegate
+ */
+ static class SynchronizedHandler implements InvocationHandler {
+ private final Object theDelegate;
+
+ SynchronizedHandler(Object theDelegate) {
+ this.theDelegate = theDelegate;
+ }
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ synchronized (theDelegate) {
+ return method.invoke(theDelegate, args);
+ }
+ }
+ }
+}

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/changelog/6_6_0/4621-work-chunk-events.yaml

@@ -0,0 +1,4 @@
+---
+type: change
+issue: 4621
+title: "Batch2 work-chunk processing now aligns transaction boundaries with event transitions."

hapi-fhir-docs/src/main/resources/ca/uhn/hapi/fhir/docs/server_jpa_batch/batch2_states.md

@@ -0,0 +1,68 @@
+
+```mermaid
+---
+title: Batch2 Job Instance state transitions
+---
+stateDiagram-v2
+ [*] --> QUEUED : on db create and queued on kakfa
+ QUEUED --> IN_PROGRESS : on any work-chunk received by worker
+ %% and (see ca.uhn.fhir.batch2.progress.InstanceProgress.getNewStatus())
+ state first_step_finished <<choice>>
+ IN_PROGRESS --> first_step_finished : When 1st step finishes
+ first_step_finished --> COMPLETED: if no chunks produced
+ first_step_finished --> IN_PROGRESS: chunks produced
+ IN_PROGRESS --> in_progress_poll : on poll \n(count acomplete/failed/errored chunks)
+ in_progress_poll --> COMPLETED : 0 failures, errored, or incomplete\n AND at least 1 chunk complete
+ in_progress_poll --> ERRORED : no failed but errored chunks
+ in_progress_poll --> FINALIZE : none failed, gated execution\n last step\n queue REDUCER chunk
+ in_progress_poll --> IN_PROGRESS : still work to do
+ %% ERRORED is just like IN_PROGRESS, but it is a one-way trip from IN_PROGRESS to ERRORED.
+ %% FIXME We could probably delete/merge this state with IS_PROCESS, and use the error count in the UI.
+ note left of ERRORED
+ Parallel to IS_PROCESS
+ end note
+ state in_progress_poll <<choice>>
+ state error_progress_poll <<choice>>
+ ERRORED --> error_progress_poll : on poll \n(count acomplete/failed/errored chunks)
+ error_progress_poll --> FAILED : any failed chunks
+ error_progress_poll --> ERRORED : no failed but errored chunks
+ error_progress_poll --> FINALIZE : none failed, gated execution\n last step\n queue REDUCER chunk
+ error_progress_poll --> COMPLETED : 0 failures, errored, or incomplete AND at least 1 chunk complete
+ state do_report <<choice>>
+ FINALIZE --> do_reduction: poll util worker marks REDUCER chunk yes or no.
+ do_reduction --> COMPLETED : success
+ do_reduction --> FAILED : fail
+ in_progress_poll --> FAILED : any failed chunks
+```
+
+```mermaid
+---
+title: Batch2 Job Work Chunk state transitions
+---
+stateDiagram-v2
+ state QUEUED
+ state on_receive <<choice>>
+ state IN_PROGRESS
+ state ERROR
+ state execute <<choice>>
+ state FAILED
+ state COMPLETED
+ direction LR
+ [*] --> QUEUED : on store
+
+ %% worker processing states
+ QUEUED --> on_receive : on receive by worker
+ on_receive --> IN_PROGRESS : start execution
+
+ IN_PROGRESS --> execute: execute
+ execute --> ERROR : on re-triable error
+ execute --> COMPLETED : success\n maybe trigger instance first_step_finished
+ execute --> FAILED : on unrecoverable \n or too many errors
+
+ %% temporary error state until retry
+ ERROR --> on_receive : exception rollback\n triggers redelivery
+
+ %% terminal states 
+ COMPLETED --> [*]
+ FAILED --> [*]
+```

hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/batch2/JpaBatch2Config.java

@@ -22,36 +22,38 @@
 
 import ca.uhn.fhir.batch2.api.IJobPersistence;
 import ca.uhn.fhir.batch2.config.BaseBatch2Config;
-import ca.uhn.fhir.batch2.coordinator.SynchronizedJobPersistenceWrapper;
 import ca.uhn.fhir.jpa.bulk.export.job.BulkExportJobConfig;
 import ca.uhn.fhir.jpa.dao.data.IBatch2JobInstanceRepository;
 import ca.uhn.fhir.jpa.dao.data.IBatch2WorkChunkRepository;
 import ca.uhn.fhir.system.HapiSystemProperties;
+import ca.uhn.fhir.util.ProxyUtil;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Primary;
 import org.springframework.transaction.PlatformTransactionManager;
 
+import javax.persistence.EntityManager;
+
 @Configuration
 @Import({
  BulkExportJobConfig.class
 })
 public class JpaBatch2Config extends BaseBatch2Config {
 
  @Bean
- public IJobPersistence batch2JobInstancePersister(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager) {
- return new JpaJobPersistenceImpl(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager);
+ public IJobPersistence batch2JobInstancePersister(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager, EntityManager theEntityManager) {
+ return new JpaJobPersistenceImpl(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager, theEntityManager);
  }
 
  @Primary
  @Bean
- public IJobPersistence batch2JobInstancePersisterWrapper(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager) {
- IJobPersistence retVal = batch2JobInstancePersister(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager);
+ public IJobPersistence batch2JobInstancePersisterWrapper(IBatch2JobInstanceRepository theJobInstanceRepository, IBatch2WorkChunkRepository theWorkChunkRepository, PlatformTransactionManager theTransactionManager, EntityManager theEntityManager) {
+ IJobPersistence retVal = batch2JobInstancePersister(theJobInstanceRepository, theWorkChunkRepository, theTransactionManager, theEntityManager);
  // Avoid H2 synchronization issues caused by
  // https://github.com/h2database/h2database/issues/1808
  if (HapiSystemProperties.isUnitTestModeEnabled()) {
- retVal = new SynchronizedJobPersistenceWrapper(retVal);
+ retVal = ProxyUtil.synchronizedProxy(IJobPersistence.class, retVal);
  }
  return retVal;
  }