Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rule apply patient export #4892

Open
wants to merge 41 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
55305ad
Force Verify tests
tadgh May 3, 2023
71edea3
fix ITs (#4809)
fil512 May 3, 2023
5855ea9
Fix migrator error on Oracle (#4814)
jamesagnew May 4, 2023
2821201
Update clinical reasoning version (#4816)
barhodes May 4, 2023
b33263d
Opening the care-gaps endpoint for GET. (#4823)
chalmarm May 5, 2023
9af24ad
added version to mdm golden resource tag (#4820)
longma1 May 5, 2023
f97da3b
Update the changelog for 4697 to be more descriptive (#4827)
JPercival May 5, 2023
609bc53
Fixes a bug with tags. (#4813)
tadgh May 5, 2023
a8e823a
fix migration issue (#4830)
fil512 May 6, 2023
5212009
Create correct version enum
tadgh May 7, 2023
ab070ef
Remove superfluous migration
tadgh May 7, 2023
5fa92dc
fixing test (#4835)
TipzCM May 8, 2023
d63dc07
email subscription, throw NullPointerException (#4790)
samguntersmilecdr May 10, 2023
37bfbf0
Add docs for CR operations (#4855)
barhodes May 11, 2023
9d9df27
Add documentation for $care-gaps operation. (#4862)
chalmarm May 11, 2023
22c9a18
4853 validation does not error when display is not the same as the di…
StevenXLi May 11, 2023
99f58f5
fixing patient everything operator (#4845)
TipzCM May 11, 2023
c3ccfb3
fix link
tadgh May 12, 2023
1891a19
Move image file
tadgh May 12, 2023
f464113
Bundle resources containing over 100 references to the same Organizat…
lukedegruchy May 12, 2023
e8cc02f
added warn message and test (#4848)
longma1 May 12, 2023
681a21c
Issue 4804 full table scan on mpi link during mdm clear (#4805)
jmarchionatto May 12, 2023
30eea1b
Fix erroneous batch 2 $export 75% complete count when the job is COMP…
lukedegruchy May 12, 2023
cf8935e
disable wars (#4877)
fil512 May 14, 2023
2973b65
4868 fix paging hapi (#4870)
TipzCM May 15, 2023
0303120
Test, fix, and changelog
michaelabuckley May 15, 2023
680dcc0
4875-binary-access-write-doest-trigger-STORAGE-BINARY-ASSIGN-BLOB-ID-…
jmarchionatto May 15, 2023
35f7b52
Better partition resolution
michaelabuckley May 15, 2023
dd56b13
Add checks based on rule applier
tadgh May 15, 2023
0a8e516
Fix ordering failure due to hash set
tadgh May 15, 2023
cf19036
Allow empty auth interceptor
tadgh May 15, 2023
75e28ba
Fix batch job (bulk export) processed record count (#4879)
michaelabuckley May 15, 2023
3c788c1
Fix up operation type on invocation
tadgh May 15, 2023
835d6d2
Throw 404 when requesting $export of non-existent Group or Patient (#…
michaelabuckley May 15, 2023
9d9dc9a
Add more tests, make hack implementation for patient instance level o…
tadgh May 15, 2023
7d0813f
Tighten test name
tadgh May 15, 2023
879e5c9
Merge branch 'rel_6_6' into rule-apply-patient-export
tadgh May 16, 2023
792e755
Changelog
tadgh May 16, 2023
f60f6aa
Default method
tadgh May 16, 2023
b989710
remove dead method
tadgh May 16, 2023
c20eab6
Remove dead autowire
tadgh May 16, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
type: fix
issue: 4875
title: "Previously, `$binary-access-write` operation didn't trigger `STORAGE_BINARY_ASSIGN_BLOB_ID_PREFIX` Pointcut.
This has been fixed."
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
---
type: fix
issue: 4878
title: "Batch jobs occasionaly reported zero (0) record processed counts. This has been corrected."
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import ca.uhn.fhir.jpa.binary.svc.BaseBinaryStorageSvcImpl;
import ca.uhn.fhir.jpa.dao.data.IBinaryStorageEntityDao;
import ca.uhn.fhir.jpa.model.entity.BinaryStorageEntity;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import com.google.common.hash.HashingInputStream;
import com.google.common.io.ByteStreams;
Expand All @@ -36,6 +37,7 @@
import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;

import javax.annotation.Nonnull;
import javax.persistence.EntityManager;
import javax.persistence.PersistenceContext;
import javax.persistence.PersistenceContextType;
Expand All @@ -55,9 +57,11 @@ public class DatabaseBlobBinaryStorageSvcImpl extends BaseBinaryStorageSvcImpl {
@Autowired
private IBinaryStorageEntityDao myBinaryStorageEntityDao;

@Nonnull
@Override
@Transactional(propagation = Propagation.REQUIRED)
public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, String theContentType, InputStream theInputStream) throws IOException {
public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, String theContentType,
InputStream theInputStream, RequestDetails theRequestDetails) throws IOException {

/*
* Note on transactionality: This method used to have a propagation value of SUPPORTS and then do the actual
Expand All @@ -70,17 +74,16 @@ public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, St
HashingInputStream hashingInputStream = createHashingInputStream(theInputStream);
CountingInputStream countingInputStream = createCountingInputStream(hashingInputStream);

String id = super.provideIdForNewBlob(theBlobIdOrNull);

BinaryStorageEntity entity = new BinaryStorageEntity();
entity.setResourceId(theResourceId.toUnqualifiedVersionless().getValue());
entity.setBlobId(id);
entity.setBlobContentType(theContentType);
entity.setPublished(publishedDate);

Session session = (Session) myEntityManager.getDelegate();
LobHelper lobHelper = session.getLobHelper();
byte[] loadedStream = IOUtils.toByteArray(countingInputStream);
String id = super.provideIdForNewBlob(theBlobIdOrNull, loadedStream, theRequestDetails, theContentType);
entity.setBlobId(id);
Blob dataBlob = lobHelper.createBlob(loadedStream);
entity.setBlob(dataBlob);

Expand All @@ -105,7 +108,7 @@ public StoredDetails storeBlob(IIdType theResourceId, String theBlobIdOrNull, St
public StoredDetails fetchBlobDetails(IIdType theResourceId, String theBlobId) {

Optional<BinaryStorageEntity> entityOpt = myBinaryStorageEntityDao.findByIdAndResourceId(theBlobId, theResourceId.toUnqualifiedVersionless().getValue());
if (entityOpt.isPresent() == false) {
if (entityOpt.isEmpty()) {
return null;
}

Expand All @@ -121,7 +124,7 @@ public StoredDetails fetchBlobDetails(IIdType theResourceId, String theBlobId) {
@Override
public boolean writeBlob(IIdType theResourceId, String theBlobId, OutputStream theOutputStream) throws IOException {
Optional<BinaryStorageEntity> entityOpt = myBinaryStorageEntityDao.findByIdAndResourceId(theBlobId, theResourceId.toUnqualifiedVersionless().getValue());
if (entityOpt.isPresent() == false) {
if (entityOpt.isEmpty()) {
return false;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package ca.uhn.fhir.jpa.batch2;

import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.api.model.Batch2JobInfo;
import ca.uhn.fhir.jpa.api.model.BulkExportJobResults;
import ca.uhn.fhir.jpa.api.model.BulkExportParameters;
import ca.uhn.fhir.jpa.api.svc.IBatch2JobRunner;
Expand Down Expand Up @@ -33,6 +34,7 @@
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ExecutionException;
Expand All @@ -44,6 +46,7 @@

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.emptyOrNullString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.hasItem;
import static org.hamcrest.Matchers.not;
import static org.junit.jupiter.api.Assertions.assertEquals;
Expand Down Expand Up @@ -181,7 +184,8 @@ private void duAbuseTest(int taskExecutions) throws InterruptedException, Execut

private void verifyBulkExportResults(String theInstanceId, List<String> theContainedList, List<String> theExcludedList) {
// Iterate over the files
String report = myJobRunner.getJobInfo(theInstanceId).getReport();
Batch2JobInfo jobInfo = myJobRunner.getJobInfo(theInstanceId);
String report = jobInfo.getReport();
ourLog.debug("Export job {} report: {}", theInstanceId, report);
if (!theContainedList.isEmpty()) {
assertThat("report for instance " + theInstanceId + " is empty", report, not(emptyOrNullString()));
Expand Down Expand Up @@ -227,6 +231,10 @@ private void verifyBulkExportResults(String theInstanceId, List<String> theConta
for (String excludedString : theExcludedList) {
assertThat("export doesn't have expected ids", foundIds, not(hasItem(excludedString)));
}

assertThat(jobInfo.getCombinedRecordsProcessed(), equalTo(2));

ourLog.info("Job {} ok", theInstanceId);
}

private String startJob(BulkDataExportOptions theOptions) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
import ca.uhn.fhir.jpa.binary.api.IBinaryStorageSvc;
import ca.uhn.fhir.jpa.binary.api.StoredDetails;
import ca.uhn.fhir.jpa.binary.provider.BinaryAccessProvider;
import ca.uhn.fhir.mdm.util.MessageHelper;
import ca.uhn.fhir.rest.api.server.RequestDetails;
import ca.uhn.fhir.rest.server.RestfulServer;
import ca.uhn.fhir.rest.server.exceptions.InvalidRequestException;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
Expand Down Expand Up @@ -39,6 +39,7 @@
import static org.mockito.ArgumentMatchers.anyBoolean;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.isNull;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
Expand Down Expand Up @@ -66,8 +67,6 @@ public class BinaryAccessProviderTest {
@Spy
protected IBinaryStorageSvc myBinaryStorageSvc;
@Autowired
private MessageHelper myMessageHelper;
@Autowired
private IInterceptorBroadcaster myInterceptorBroadcaster;


Expand Down Expand Up @@ -157,7 +156,7 @@ public void testBinaryAccessRead_WithoutAttachmentId() throws IOException {
}

@Test
public void testBinaryAccessRead_WithoutAttachmentId_NullData() throws IOException {
public void testBinaryAccessRead_WithoutAttachmentId_NullData() {
DocumentReference docRef = new DocumentReference();
DocumentReference.DocumentReferenceContentComponent content = docRef.addContent();
content.getAttachment().setContentType("application/octet-stream");
Expand Down Expand Up @@ -257,7 +256,7 @@ public void testBinaryAccessWrite_WithContentLength15() throws IOException {
when(theServletRequest.getContentLength()).thenReturn(15);
when(myBinaryStorageSvc.shouldStoreBlob(15, docRef.getIdElement(), "Integer")).thenReturn(true);
myRequestDetails.setServletRequest(theServletRequest);
when(myBinaryStorageSvc.storeBlob(eq(docRef.getIdElement()), isNull(), eq("Integer"), any(InputStream.class))).thenReturn(sd);
doReturn(sd).when(myBinaryStorageSvc).storeBlob(eq(docRef.getIdElement()), isNull(), eq("Integer"), any(InputStream.class), any(RequestDetails.class));
myRequestDetails.setRequestContents(SOME_BYTES);

try {
Expand All @@ -266,7 +265,7 @@ public void testBinaryAccessWrite_WithContentLength15() throws IOException {
assertEquals(docRef.getId(), outcome.getIdElement().getValue());
} catch (IOException e) {
}
verify(myBinaryStorageSvc, times(1)).storeBlob(any(), any(), any(), any());
verify(myBinaryStorageSvc, times(1)).storeBlob(any(), any(), any(), any(), any(ServletRequestDetails.class));
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import ca.uhn.fhir.jpa.model.entity.BinaryStorageEntity;
import ca.uhn.fhir.jpa.test.BaseJpaR4Test;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import org.hl7.fhir.r4.model.IdType;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
Expand Down Expand Up @@ -52,7 +53,7 @@ public void testStoreAndRetrieve() throws IOException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(SOME_BYTES);
String contentType = "image/png";
IdType resourceId = new IdType("Binary/123");
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream);
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream, new ServletRequestDetails());

myCaptureQueriesListener.logAllQueriesForCurrentThread();

Expand Down Expand Up @@ -105,7 +106,7 @@ public void testStoreAndRetrieveWithManualId() throws IOException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(SOME_BYTES);
String contentType = "image/png";
IdType resourceId = new IdType("Binary/123");
StoredDetails outcome = mySvc.storeBlob(resourceId, "ABCDEFG", contentType, inputStream);
StoredDetails outcome = mySvc.storeBlob(resourceId, "ABCDEFG", contentType, inputStream, new ServletRequestDetails());
assertEquals("ABCDEFG", outcome.getBlobId());

myCaptureQueriesListener.logAllQueriesForCurrentThread();
Expand Down Expand Up @@ -163,7 +164,7 @@ public void testExpunge() throws IOException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(SOME_BYTES);
String contentType = "image/png";
IdType resourceId = new IdType("Binary/123");
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream);
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream, new ServletRequestDetails());
String blobId = outcome.getBlobId();

// Expunge
Expand All @@ -185,7 +186,7 @@ public void testWrongResourceId() throws IOException {
ByteArrayInputStream inputStream = new ByteArrayInputStream(SOME_BYTES);
String contentType = "image/png";
IdType resourceId = new IdType("Binary/123");
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream);
StoredDetails outcome = mySvc.storeBlob(resourceId, null, contentType, inputStream, new ServletRequestDetails());

// Right ID
ByteArrayOutputStream capture = new ByteArrayOutputStream();
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package ca.uhn.fhir.jpa.binstore;

import ca.uhn.fhir.context.FhirContext;
import ca.uhn.fhir.i18n.Msg;
import ca.uhn.fhir.interceptor.executor.InterceptorService;
import ca.uhn.fhir.jpa.binary.api.StoredDetails;
import ca.uhn.fhir.rest.server.exceptions.PayloadTooLargeException;
import ca.uhn.fhir.rest.server.exceptions.ResourceNotFoundException;
import ca.uhn.fhir.rest.server.servlet.ServletRequestDetails;
import org.apache.commons.io.FileUtils;
import org.hl7.fhir.instance.model.api.IIdType;
import org.hl7.fhir.r4.model.IdType;
Expand Down Expand Up @@ -34,6 +37,8 @@ public class FilesystemBinaryStorageSvcImplTest {
public void before() {
myPath = new File("./target/fstmp");
mySvc = new FilesystemBinaryStorageSvcImpl(myPath.getAbsolutePath());
mySvc.setFhirContextForTests(FhirContext.forR4Cached());
mySvc.setInterceptorBroadcasterForTests(new InterceptorService());
}

@AfterEach
Expand All @@ -45,7 +50,7 @@ public void after() throws IOException {
public void testStoreAndRetrieve() throws IOException {
IIdType id = new IdType("Patient/123");
String contentType = "image/png";
StoredDetails outcome = mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES));
StoredDetails outcome = mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES), new ServletRequestDetails());

ourLog.info("Got id: {}", outcome);

Expand All @@ -68,7 +73,7 @@ public void testStoreAndRetrieveManualId() throws IOException {
IIdType id = new IdType("Patient/123");
String contentType = "image/png";
String blobId = "ABCDEFGHIJKLMNOPQRSTUV";
StoredDetails outcome = mySvc.storeBlob(id, blobId, contentType, new ByteArrayInputStream(SOME_BYTES));
StoredDetails outcome = mySvc.storeBlob(id, blobId, contentType, new ByteArrayInputStream(SOME_BYTES), new ServletRequestDetails());
assertEquals(blobId, outcome.getBlobId());

ourLog.info("Got id: {}", outcome);
Expand Down Expand Up @@ -103,7 +108,7 @@ public void testFetchBlobUnknown() throws IOException {
public void testExpunge() throws IOException {
IIdType id = new IdType("Patient/123");
String contentType = "image/png";
StoredDetails outcome = mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES));
StoredDetails outcome = mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES), new ServletRequestDetails());

ourLog.info("Got id: {}", outcome);

Expand All @@ -129,7 +134,7 @@ public void testRejectOversized() throws IOException {
IIdType id = new IdType("Patient/123");
String contentType = "image/png";
try {
mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES));
mySvc.storeBlob(id, null, contentType, new ByteArrayInputStream(SOME_BYTES), new ServletRequestDetails());
fail();
} catch (PayloadTooLargeException e) {
assertEquals(Msg.code(1343) + "Binary size exceeds maximum: 5", e.getMessage());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,43 +18,31 @@ public void shouldStoreBlob() {

@Test
public void storeBlob() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.storeBlob(null, null, null, null);
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.storeBlob(null, null, null, null, null));
}

@Test
public void fetchBlobDetails() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.fetchBlobDetails(null, null);
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.fetchBlobDetails(null, null));
}

@Test
public void writeBlob() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.writeBlob(null, null, null);
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.writeBlob(null, null, null));
}

@Test
public void expungeBlob() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.expungeBlob(null, null);
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.expungeBlob(null, null));
}

@Test
public void fetchBlob() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.fetchBlob(null, null);
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.fetchBlob(null, null));
}

@Test
public void newBlobId() {
assertThrows(UnsupportedOperationException.class, () -> {
mySvc.newBlobId();
});
assertThrows(UnsupportedOperationException.class, () -> mySvc.newBlobId(null, null, null));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import ca.uhn.fhir.interceptor.model.RequestPartitionId;
import ca.uhn.fhir.jpa.api.config.JpaStorageSettings;
import ca.uhn.fhir.jpa.api.dao.DaoRegistry;
import ca.uhn.fhir.jpa.api.dao.IFhirResourceDao;
import ca.uhn.fhir.jpa.api.model.Batch2JobInfo;
import ca.uhn.fhir.jpa.api.model.Batch2JobOperationResult;
import ca.uhn.fhir.jpa.api.model.BulkExportJobResults;
Expand Down Expand Up @@ -98,6 +99,8 @@ public class BulkDataExportProviderTest {
private final HttpClientExtension myClient = new HttpClientExtension();
@Mock
private IBatch2JobRunner myJobRunner;
@Mock
IFhirResourceDao myFhirResourceDao;
@InjectMocks
private BulkDataExportProvider myProvider;
@RegisterExtension
Expand Down Expand Up @@ -140,6 +143,8 @@ public void injectStorageSettings() {
myProvider.setStorageSettings(myStorageSettings);
DaoRegistry daoRegistry = mock(DaoRegistry.class);
lenient().when(daoRegistry.getRegisteredDaoTypes()).thenReturn(Set.of("Patient", "Observation", "Encounter"));

lenient().when(daoRegistry.getResourceDao(anyString())).thenReturn(myFhirResourceDao);
myProvider.setDaoRegistry(daoRegistry);

}
Expand Down
Loading
Loading