Merge pull request #976 from AtlasOfLivingAustralia/feature/issue929

Feature/issue929

grails-app/conf/application.groovy

@@ -566,6 +566,7 @@ if (!grails.cache.ehcache) {
  }
  }
 }
+ehcache.directory='/data/ecodata/ehcache'
 
 
 security {
@@ -623,20 +624,11 @@ environments {
  app.uploads.url = "/document/download/"
  grails.mail.host="localhost"
  grails.mail.port=1025
- grails.cache.ehcache = {
- diskStore {
- path "~/data/${appName}/ehcache"
- }
- }
+ ehcache.directory="./ehcache"
 
  }
  test {
- // Override disk store so the travis build doesn't fail.
- grails.cache.ehcache = {
- diskStore {
- path '/tmp'
- }
- }
+ ehcache.directory="./ehcache"
  grails.logging.jul.usebridge = true
  ecodata.use.uuids = false
  app.external.model.dir = "./models/"
@@ -661,11 +653,7 @@ environments {
  security.cas.loginUrl="${security.cas.casServerUrlPrefix}/login"
  }
  meritfunctionaltest {
- grails.cache.ehcache = {
- diskStore {
- path '/tmp'
- }
- }
+ ehcache.directory="./ehcache"
  security.cas.bypass = true
  grails.logging.jul.usebridge = true
  ecodata.use.uuids = false

grails-app/conf/ecodata-ehcache.xml

@@ -2,7 +2,7 @@
  xmlns='http:https://www.ehcache.org/v3'
  xmlns:jsr107='http:https://www.ehcache.org/v3/jsr107'>
 
- <persistence directory="/data/ecodata/ehcache"/>
+ <persistence directory="${ehcache.directory}"/>
 
  <cache alias="spatialGeoJsonPid" uses-template="defaultSetting">
  <expiry>
@@ -38,6 +38,21 @@
  <heap unit="entries">100</heap>
  </resources>
  </cache>
+ <cache alias="spatialPidObjectGeometry">
+
+ <key-type>java.io.Serializable</key-type>
+ <value-type>java.io.Serializable</value-type>
+
+ <expiry>
+ <tti unit="days">365</tti>
+ </expiry>
+ <resources>
+ <heap unit="entries">100</heap>
+
+ <disk unit="MB" persistent="true">500</disk>
+ </resources>
+ </cache>
+
  <cache alias="grailsTemplatesCache">
  <key-type>java.io.Serializable</key-type>
  <value-type>java.io.Serializable</value-type>
@@ -54,6 +69,17 @@
  </expiry>
  <heap>10</heap>
  </cache>
+
+ <cache alias="userDetailsCache">
+
+ <expiry>
+ <ttl unit="days">1</ttl>
+ </expiry>
+ <resources>
+ <heap unit="entries">2000</heap>
+ </resources>
+ </cache>
+
  <cache-template name="defaultSetting">
  <key-type>java.io.Serializable</key-type>
  <value-type>java.util.HashMap</value-type>

grails-app/conf/logback.xml

@@ -11,7 +11,9 @@
  </encoder>
  </appender>
 
- <logger name="au.org.ala.ecodata" level="DEBUG" />
+ <logger name="au.org.ala.ecodata" level="INFO" />
+ <logger name="au.org.ala.ecodata.SpatialService" level="DEBUG"/>
+ <logger name="org.elasticsearch.client" level="ERROR" />
 
  <logger name="org.ehcache" level="DEBUG" />
 

grails-app/init/au/org/ala/ecodata/Application.groovy

@@ -4,6 +4,7 @@ import au.org.ala.userdetails.UserDetailsClient
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.Rfc3339DateJsonAdapter
 import grails.boot.GrailsApp
+import grails.boot.config.GrailsApplicationPostProcessor
 import grails.boot.config.GrailsAutoConfiguration
 import grails.core.GrailsApplication
 import graphql.Scalars
@@ -30,6 +31,9 @@ import org.springframework.context.annotation.ComponentScan
 //@Slf4j
 class Application extends GrailsAutoConfiguration {
 
+ private static final String EHCACHE_DIRECTORY_CONFIG_ITEM = "ehcache.directory"
+ private static final String DEFAULT_EHCACHE_DIRECTORY = "./ehcache"
+
  static void main(String[] args) {
  GrailsApp.run(Application, args)
  }
@@ -48,4 +52,22 @@ class Application extends GrailsAutoConfiguration {
  SchemaGenerator schemaGenerator = new SchemaGenerator();
  return schemaGenerator.makeExecutableSchema(typeDefinitionRegistry, runtimeWiring);
  }
+
+ @Bean
+ GrailsApplicationPostProcessor grailsApplicationPostProcessor() {
+
+ // We are overriding the GrailsApplicationPostProcessor because we need a lifecycle hook after
+ // the configuration has been read, but before the plugin lifecycle bean initialisation has started.
+ // This is because the grails ehcache plugin only supports configuration via XML files and the
+ // cache directory store can only be configured via an environment variable.
+ // To keep the configuration in one place, we are reading the config, and setting the system property
+ // so it can be read during cache initialisation.
+ return new GrailsApplicationPostProcessor( this, applicationContext, classes() as Class[]) {
+ @Override
+ protected void customizeGrailsApplication(GrailsApplication grailsApplication) {
+ System.setProperty(EHCACHE_DIRECTORY_CONFIG_ITEM, grailsApplication.config.getProperty(EHCACHE_DIRECTORY_CONFIG_ITEM, DEFAULT_EHCACHE_DIRECTORY))
+ }
+
+ }
+ }
 }

grails-app/services/au/org/ala/ecodata/SiteService.groovy

@@ -860,11 +860,23 @@ class SiteService {
  geographicFacets = spatialService.intersectPid(site.extent.geometry.pid as String, fid, fidsToLookup)
  break
  default:
- Map geom = geometryAsGeoJson(site)
- geographicFacets = spatialService.intersectGeometry(geom, fidsToLookup)
+ if (site.type == Site.TYPE_COMPOUND) {
+ geographicFacets = [:].withDefault{[]}
+ site.features.each { Map feature ->
+ Map featureIntersection = spatialService.intersectGeometry(feature.geometry, fidsToLookup)
+ featureIntersection.each{k, v ->
+ geographicFacets[k] += v
+ geographicFacets[k] = geographicFacets[k].unique()
+ }
+ }
+ }
+ else {
+ Map geom = geometryAsGeoJson(site)
+ geographicFacets = spatialService.intersectGeometry(geom, fidsToLookup)
+ }
  break
  }
- geographicFacets
+ geographicFacets ?: [:]
 
  }
 

grails-app/services/au/org/ala/ecodata/SpatialService.groovy

@@ -4,7 +4,11 @@ import grails.core.GrailsApplication
 import grails.plugin.cache.Cacheable
 import groovy.json.JsonParserType
 import groovy.json.JsonSlurper
+import org.locationtech.jts.geom.Coordinate
 import org.locationtech.jts.geom.Geometry
+import org.locationtech.jts.geom.GeometryFactory
+import org.locationtech.jts.io.WKTReader
+
 import static ParatooService.deepCopy
 /**
  * The SpatialService is responsible for:
@@ -42,8 +46,8 @@ class SpatialService {
  Map<String,List<String>> intersectGeometry(Map geoJson, List<String> fieldIds = null) {
  int length = geoJson?.toString().size()
  int threshold = grailsApplication.config.getProperty('spatial.geoJsonEnvelopeConversionThreshold', Integer)
+ Geometry geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
  if(length > threshold){
- Geometry geo = GeometryUtils.geoJsonMapToGeometry (geoJson)
  geoJson = GeometryUtils.geometryToGeoJsonMap (geo.getEnvelope())
  }
 
@@ -52,21 +56,35 @@ class SpatialService {
  fieldIds = metadataService.getSpatialLayerIdsToIntersect()
  }
 
+ long start = System.currentTimeMillis()
  // We are using a WKT string instead of geojson as the spatial portal validates geojson - using
  // WKT allows us to get away with self intersecting polygons that users occasionally draw.
- String wkt = GeometryUtils.geoJsonMapToGeometry(geoJson).toText()
+ String wkt = geo.toText()
+ long end = System.currentTimeMillis()
+ log.info("Time taken to convert geojson to wkt: ${end-start}ms")
+
 
  Map result = [:]
  fieldIds.each { fid ->
+ start = end
  Map response = webService.doPost(url+fid, wkt)
  if (response.resp && !response.error) {
  result[fid] = response.resp
  }
+ end = System.currentTimeMillis()
+ log.info("Time taken to intersect with layer $fid: ${end-start}ms")
  }
 
- filterOutObjectsInBoundary(result, geoJson)
- convertResponsesToGeographicFacets(result)
+ start = end
+ filterOutObjectsInBoundary(result, geo)
+ end = System.currentTimeMillis()
+ log.info("Time taken to filter out objects in boundary: ${end-start}ms")
 
+ start = end
+ Map geographicFacets = convertResponsesToGeographicFacets(result)
+ end = System.currentTimeMillis()
+ log.info("Time taken to convert responses to geographic facets: ${end-start}ms")
+ geographicFacets
  }
 
  /**
@@ -104,15 +122,20 @@ class SpatialService {
  convertResponsesToGeographicFacets(result)
  }
 
+ private void filterOutObjectsInBoundary(Map response, Map mainObjectGeoJson) {
+ Geometry mainGeometry = GeometryUtils.geoJsonMapToGeometry(mainObjectGeoJson)
+ filterOutObjectsInBoundary(response, mainGeometry)
+ }
+
  /**
  * Spatial portal intersection returns values at the boundary. This function filters the boundary intersection by
  * comparing if the area of intersection is less than a predefined amount 5% (this is configurable).
  * NOTE: GeoJSON objects must be valid for filtering out to work.
  * @param response - per layer/fid intersection values - [ "cl34" : [[pid: 123, name: "ACT", fid: "cl34", id: "ACT" ...], ...]
  * @param mainObjectGeoJson - GeoJSON object that is used to intersect with layers.
  */
- void filterOutObjectsInBoundary(Map response, Map mainObjectGeoJson) {
- Geometry mainGeometry = GeometryUtils.geoJsonMapToGeometry(mainObjectGeoJson)
+ private void filterOutObjectsInBoundary(Map response, Geometry mainGeometry) {
+
  if (!mainGeometry.isValid()) {
  log.info("Main geometry invalid. Cannot check intersection is near boundary.")
  return
@@ -123,16 +146,27 @@ class SpatialService {
  matchingObjects.each { Map obj ->
  String boundaryPid = obj.pid
  if (boundaryPid) {
+ log.debug("Intersecting ${obj.fieldname}(${fid}) - ${obj.name} ")
  // Get geoJSON of the object stored in spatial portal
- def boundaryGeoJson = getGeoJsonForPidToMap(boundaryPid)
- Geometry boundaryGeometry = GeometryUtils.geoJsonMapToGeometry(boundaryGeoJson)
+ long start = System.currentTimeMillis()
+
+ Geometry boundaryGeometry = getGeometryForPid(boundaryPid)
+ long end = System.currentTimeMillis()
+ log.debug("Time taken to convert geojson to geometry for pid $boundaryPid: ${end-start}ms")
+
  if (boundaryGeometry.isValid()) {
  // check if intersection should be ignored
- if (!isValidGeometryIntersection(mainGeometry, boundaryGeometry))
+ start = end
+ if (!isValidGeometryIntersection(mainGeometry, boundaryGeometry)) {
  pidToFilter.add(boundaryPid)
+ log.debug("Filtered out ${obj.fieldname}(${fid}) - ${obj.name}")
+ }
+
+ end = System.currentTimeMillis()
+ log.debug("Time taken to check intersection for pid $boundaryPid: ${end-start}ms")
  }
  else {
- log.info ("Cannot check object $boundaryPid($fid) is near main geomerty")
+ log.debug ("Cannot check object $boundaryPid($fid) is near main geomerty")
  }
  }
  }
@@ -205,16 +239,37 @@ class SpatialService {
 
  @Cacheable(value = "spatialGeoJsonPidObject")
  Map getGeoJsonForPidToMap(String pid) {
+ log.debug("Cache miss for getGeoJsonForPidToMap($pid)")
  getGeoJsonForPid(pid)
  }
 
+ @Cacheable(value = "spatialPidObjectGeometry", key={pid})
+ Geometry getGeometryForPid(String pid) {
+ log.debug("Cache miss for getGeometryForPid($pid)")
+ String url = grailsApplication.config.getProperty('spatial.baseUrl')+"/ws/shapes/wkt/$pid"
+ String wkt = webService.get(url)
+
+ Geometry geometry = null
+ try {
+ geometry = new WKTReader().read(wkt)
+ }
+ catch (Exception e) {
+ log.error("Error reading geometry for pid $pid")
+ // Ehcache throws an error if a null value is returned, so we create a dummy geometry
+ // that won't intersect with anything.
+ geometry = new GeometryFactory().createPoint(new Coordinate(0, 0))
+ }
+ geometry
+ }
+
  /**
  * Get GeoJSON of a spatial object.
  * @param pid
  * @return
  */
  @Cacheable(value="spatialGeoJsonPid", key= {pid})
  Map getGeoJsonForPid (String pid) {
+ log.debug("Cache miss for getGeoJsonForPid($pid)")
  String url = grailsApplication.config.getProperty('spatial.baseUrl')+"/ws/shapes/geojson/$pid"
  Map resp = webService.getJson(url)
 

src/test/groovy/au/org/ala/ecodata/SiteServiceSpec.groovy

@@ -330,16 +330,19 @@ class SiteServiceSpec extends MongoSpec implements ServiceUnitTest<SiteService>
  service.populateLocationMetadataForSite(site)
 
  then:
- 1 * spatialServiceMock.intersectGeometry(_, _) >> [:]
+ 2 * spatialServiceMock.intersectGeometry(_, _) >> ["electorate":["Bean"], "state":["ACT"]]
  site.extent.geometry.aream2 == 4938.9846950349165d
+ site.extent.geometry.electorate == ["Bean"]
+ site.extent.geometry.state == ["ACT"]
 
  when:
  site.type = Site.TYPE_WORKS_AREA
  service.populateLocationMetadataForSite(site)
 
- then:
- 1 * spatialServiceMock.intersectGeometry(_, _) >> [:]
+ then: "Each feature is intersected individually and duplicates removed"
+ 1 * spatialServiceMock.intersectGeometry(_, _) >> ["state":["ACT"]]
  site.extent.geometry.aream2 == 2469.492347517461
+ site.extent.geometry.state == ["ACT"]
 
  }
 

src/test/groovy/au/org/ala/ecodata/SpatialServiceSpec.groovy

@@ -58,7 +58,7 @@ class SpatialServiceSpec extends Specification implements ServiceUnitTest<Spatia
  then:
  response["cl22"].size() == 1
  response["cl22"][0].name == "ACT"
- 1 * webService.getJson("/ws/shapes/geojson/123") >> [type: "Polygon", coordinates: [[[5, 0], [5, 5], [0, 5], [0, 0], [5, 0]]]]
- 1 * webService.getJson("/ws/shapes/geojson/456") >> [type: "Polygon", coordinates: [[[11.5, 9.5], [11.5, 11.5], [9.5, 11.5], [9.5, 9.5], [11.5, 9.5]]]]
+ 1 * webService.get("/ws/shapes/wkt/123") >> GeometryUtils.geoJsonMapToGeometry([type: "Polygon", coordinates: [[[5, 0], [5, 5], [0, 5], [0, 0], [5, 0]]]])
+ 1 * webService.get("/ws/shapes/wkt/456") >> GeometryUtils.geoJsonMapToGeometry([type: "Polygon", coordinates: [[[11.5, 9.5], [11.5, 11.5], [9.5, 11.5], [9.5, 9.5], [11.5, 9.5]]]])
  }
 }