Factor out service protocol. Rename expression. Tidying.

.clj-kondo/config.edn

@@ -0,0 +1,10 @@
+{:lint-as {io.pedestal.interceptor.error/error-dispatch clj-kondo.lint-as/def-catch-all}
+
+ :linters {:unsorted-required-namespaces {:level :warning}
+ :invalid-arity {:level :error}
+ :missing-else-branch {:level :off}
+
+ :unused-namespace {:level :warning
+ :exclude []}
+ :unused-referred-var {:level :warning
+ :exclude {}}}}

.gitignore

@@ -5,4 +5,5 @@ classes/
 target/
 snomed.db
 search.db
-hermes.iml
+hermes.iml
+.clj-kondo/.cache

src/com/eldrix/hermes/expression.clj → src/com/eldrix/hermes/cg.clj

@@ -1,12 +1,14 @@
-(ns com.eldrix.hermes.expression
- (:require [clojure.java.io :as io]
-  [clojure.string :as str]
+(ns com.eldrix.hermes.cg
+ "Support for SNOMED CT compositional grammar."
+ (:require [clojure.data.zip.xml :as zx]
  [clojure.edn :as edn]
- [instaparse.core :as insta]
- [clojure.data.zip.xml :as zx]
+ [clojure.java.io :as io]
+ [clojure.string :as str]
  [clojure.walk :as walk]
  [clojure.zip :as zip]
- [com.eldrix.hermes.impl.store :as store]))
+ [com.eldrix.hermes.impl.language :as lang]
+ [com.eldrix.hermes.impl.store :as store]
+ [instaparse.core :as insta]))
 
 (def cg-parser
  (insta/parser (io/resource "cg-v2.4.abnf") :input-format :abnf :output-format :enlive))
@@ -128,10 +130,10 @@
  [config concept]
  (str (:conceptId concept)
  (when-not (:hide-terms? config)
- (let [has-term (or (:term concept) (:getPreferredSynonym config))]
+ (let [has-term (or (:term concept) (:get-preferred-synonym-fn config))]
  (str
  (when has-term "|")
- (if-let [f (:getPreferredSynonym config)]
+ (if-let [f (:get-preferred-synonym-fn config)]
  (or (:term (f (:conceptId concept))) (:term concept))
  (:term concept))
  (when has-term "|"))))))
@@ -173,39 +175,17 @@
 (defn render
  "Render an expression into string form.
  Parameters:
- - store : SNOMED store
+ - st  : SNOMED store
  - hide-terms? : do not include textual terms in output
  - update-terms? : update terms for the preferred synonyms in locale(s) specified.
  - locale-priorities : list of locale priorities (e.g. \"en-GB\")"
- [{:keys [store hide-terms? update-terms? locale-priorities] :as config} exp]
- (let [cfg (if (and store update-terms? locale-priorities)
- (if-let [langs (store/ordered-language-refsets-from-locale locale-priorities (store/get-installed-reference-sets store))]
- (assoc config :getPreferredSynonym #(store/get-preferred-synonym store % langs))
- config)
+ [{:keys [store update-terms? locale-priorities] :as config} exp]
+ (let [lang-refsets (when store (lang/match store locale-priorities))
+ cfg (if (and store update-terms? (seq lang-refsets))
+ (assoc config :get-preferred-synonym-fn (fn [concept-id] (store/get-preferred-synonym store concept-id lang-refsets)))
  config)]
  (str (:definitionStatus exp) " " (render-subexpression cfg (:subExpression exp)))))
 
 
 (comment
- (def p (parse "24700007"))
- (def st (store/open-store "snomed.db"))
- (render {:store st :update-terms? true :locale-priorities "en-GB"} p)
- (def p (parse "80146002|appendectomy|:260870009|priority|=25876001|emergency|, 425391005|using access device|=86174004|laparoscope|"))
- (render {:store st :update-terms? true :locale-priorities "en-GB"} p)
- (simplify p)
- (def p (parse " 71388002 |procedure| :
- { 260686004 |method| = 129304002 |excision - action| ,
- 405813007 |procedure site - direct| = 20837000 |structure of right ovary| ,
- 424226004 |using device| = 122456005 |laser device| }
- { 260686004 |method| = 261519002 |diathermy excision - action| ,
- 405813007 |procedure site - direct| = 113293009 |structure of left fallopian tube| }"))
- (def p (parse " 91143003 |Albuterol| :
- 411116001 |Has manufactured dose form| = 385023001 |oral solution| ,
- { 127489000 |Has active ingredient| = 372897005 |Albuterol| ,
- 179999999100 |Has basis of strength| = 372897005 |Albuterol| ,
- 189999999103 |Has strength value| = #0.083, 199999999101 |Has strength numerator unit| = 118582008 |%| }"))
-
-
- (def rs (get-in p [:subExpression :refinements]))
-
  )

src/com/eldrix/hermes/config.clj

@@ -1,10 +1,11 @@
 (ns com.eldrix.hermes.config
  (:require [aero.core :as aero]
  [clojure.java.io :as io]
- [integrant.core :as ig]
+ [com.eldrix.hermes.server :as server]
+ [com.eldrix.hermes.service :as svc]
  [com.eldrix.hermes.terminology :as terminology]
- [com.eldrix.hermes.server :as server])
- (:import (com.eldrix.hermes.terminology SnomedService)))
+ [integrant.core :as ig])
+ (:import (com.eldrix.hermes.service SnomedService)))
 
 (defmethod ig/init-key :terminology/service [_ {:keys [path]}]
  (terminology/open path))

src/com/eldrix/hermes/core.clj

@@ -2,12 +2,12 @@
  (:gen-class)
  (:require [clojure.pprint :as pp]
  [clojure.string :as str]
- [clojure.tools.logging :as log]
  [clojure.tools.cli :as cli]
- [integrant.core :as ig]
- [com.eldrix.hermes.import :as import]
+ [clojure.tools.logging :as log]
  [com.eldrix.hermes.config :as config]
- [com.eldrix.hermes.terminology :as terminology]))
+ [com.eldrix.hermes.import :as import]
+ [com.eldrix.hermes.terminology :as terminology]
+ [integrant.core :as ig]))
 
 (defn import-from [{:keys [db]} args]
  (if db
@@ -40,7 +40,7 @@
  (pp/pprint (terminology/get-status db))
  (log/error "no database directory specified")))
 
-(defn serve [{:keys [db port]} args]
+(defn serve [{:keys [db port]} _]
  (if db
  (let [conf (-> (:ig/system (config/config :live))
  (assoc-in [:terminology/service :path] db)
@@ -110,10 +110,5 @@
  :else (invoke-command command options (rest arguments)))))
 
 (comment
- (def filename "/Users/mark/Downloads/uk_sct2cl_30.0.0_20200805000001/SnomedCT_InternationalRF2_PRODUCTION_20190731T120000Z")
- (def filename "C:\\Users\\mark\\Dev\\downloads\\uk_sct2cl_30.0.0_20200805000001")
-
- (def svc (terminology/open "snomed.db"))
- (terminology/getConcept svc 24700007)
 
  )

src/com/eldrix/hermes/impl/language.clj

@@ -0,0 +1,70 @@
+(ns com.eldrix.hermes.impl.language
+ (:require [com.eldrix.hermes.impl.store :as store])
+ (:import (java.util Locale$LanguageRange Locale)))
+
+(def language-reference-sets
+ "Defines a mapping between ISO language tags and the language reference sets
+ (possibly) installed as part of SNOMED CT. These can be used if a client
+ does not wish to specify a reference set (or set of reference sets) to be used
+ to derive preferred or acceptable terms, but instead wants some sane defaults
+ on the basis of locale as defined by IETF BCP 47.
+
+ TODO: add more well-known mappings and ordered reference sets representing
+ those locales."
+ {"en-GB" [999001261000000100 ;; NHS realm language (clinical part)
+ 999000691000001104 ;; NHS realm language (pharmacy part)
+ 900000000000508004 ;; Great Britain English language reference set
+ ]
+ "en-US" [900000000000509007]})
+
+
+
+;;;;
+;;;;
+;;;;
+;;;;
+;;;;
+;;;;
+
+(defn- filter-language-reference-sets
+ "Filters the mapping from ISO language tags to refsets keeping only those
+ with installed reference sets."
+ [mapping installed]
+ (reduce-kv (fn [m k v]
+ (if-let [filtered (seq (filter installed v))]
+ (assoc m (Locale/forLanguageTag k) filtered)
+ (dissoc m k)))
+ {}
+ mapping))
+
+(defn- do-match
+ [installed-language-reference-sets language-priority-list]
+ (let [installed-locales (keys installed-language-reference-sets) ;; list of java.util.Locales
+ priority-list (try (Locale$LanguageRange/parse language-priority-list) (catch Exception _ []))
+ filtered (Locale/filter priority-list installed-locales)]
+ (mapcat #(get installed-language-reference-sets %) filtered)))
+
+(defn match-fn
+ "Generate a locale matching function to return the best refsets to use given a
+ 'language-priority-list'.
+ Parameters:
+ - store : a SNOMED store
+
+ Returns:
+ - a function that takes a single string containing a list of comma-separated
+ language ranges or a list of language ranges in the form of the
+ \"Accept-Language\" header defined in RFC 2616.
+
+ This closes over the installed reference sets at the time of function
+ generation and so does not take into account of changes made since
+ it was generated."
+ [store]
+ (let [installed-refsets (store/get-installed-reference-sets store)
+ installed-language-reference-sets (filter-language-reference-sets language-reference-sets installed-refsets )]
+ (partial do-match installed-language-reference-sets)))
+
+(defn match
+ "Convenience method to match a language-priority-list to the installed
+ language reference sets in the store specified."
+ [store language-priority-list]
+ ((match-fn store) language-priority-list))

src/com/eldrix/hermes/impl/search.clj

@@ -3,6 +3,7 @@
  [clojure.core.async :as async]
  [clojure.string :as str]
  [clojure.tools.logging.readable :as log]
+ [com.eldrix.hermes.impl.language :as lang]
  [com.eldrix.hermes.impl.store :as store]
  [com.eldrix.hermes.snomed :as snomed])
  (:import (org.apache.lucene.index Term IndexWriter IndexWriterConfig DirectoryReader IndexWriterConfig$OpenMode IndexReader)
@@ -88,13 +89,13 @@
 
 (defn build-search-index
  "Build a search index using the SNOMED CT store at `store-filename`."
- [store-filename search-filename locale-priorities]
+ [store-filename search-filename language-priority-list]
  (let [ch (async/chan 1 (partition-all 1000))] ;; chunk concepts into batches
  (with-open [store (store/open-store store-filename)
  writer (open-index-writer search-filename)]
- (let [langs (store/ordered-language-refsets-from-locale locale-priorities (store/get-installed-reference-sets store))]
+ (let [langs (lang/match store language-priority-list)]
  (when-not (seq langs) (throw (ex-info "No language refset for any locale listed in priority list"
- {:priority-list locale-priorities :store-filename store-filename})))
+ {:priority-list language-priority-list :store-filename store-filename})))
  (store/stream-all-concepts store ch) ;; start streaming all concepts
  (async/<!! ;; block until pipeline complete
  (async/pipeline ;; pipeline for side-effects
@@ -208,30 +209,4 @@
  (do-search searcher (assoc params :fuzzy fallback)))))))
 
 (comment
- (build-search-index "snomed.db" "search.db" "en-GB")
- (def searcher (IndexSearcher. (open-index-reader "snomed.db/search.db")))
-
- (create-test-search "snomed.db" "test-search.db")
- (def searcher (IndexSearcher. (open-index-reader "test-search.db")))
- (def store (store/open-store "snomed.db"))
- (def diabetes (store/get-concept store 73211009))
- (def langs (store/ordered-language-refsets-from-locale "en-GB" (store/get-installed-reference-sets store)))
- (do-search searcher {:s "IIH" :fuzzy 0 :fallback-fuzzy 1 :inactive-descriptions? true :properties {snomed/IsA snomed/ClinicalFinding}})
-
-
- ;; is-a 14679004 = occupations
- (do-search searcher {:s "consultant" :fuzzy 0 :fallback-fuzzy 1 :properties {snomed/IsA [14679004]}})
-
- ;;370135005 |Pathological process (attribute)| = 441862004
- (time (def one (into #{} (map :concept-id (do-search searcher {:max-hits 10000 :properties {snomed/PathologicalProcess 441862004}})))))
-
- (def two (store/get-all-children store 441862004 snomed/PathologicalProcess))
- (time (def three (into #{} (mapcat #(store/get-all-children store %) two))))
- (count one)
- (count two)
- (count three)
- (time (map :term (map (partial store/get-fully-specified-name store) three)))
-
- (store/get-fully-specified-name store 441862004)
-
  )

src/com/eldrix/hermes/impl/store.clj

@@ -1,14 +1,14 @@
 (ns com.eldrix.hermes.impl.store
  "Store provides access to a key value store."
- (:require [clojure.java.io :as io]
+ (:require [clojure.core.async :as async]
+ [clojure.java.io :as io]
  [clojure.set :as set]
- [clojure.core.async :as async]
  [clojure.tools.logging.readable :as log]
  [com.eldrix.hermes.snomed :as snomed])
  (:import [java.io FileNotFoundException Closeable]
  (org.mapdb Serializer BTreeMap DB DBMaker)
  (org.mapdb.serializer SerializerArrayTuple)
- (java.util NavigableSet Locale$LanguageRange Locale)
+ (java.util NavigableSet)
  (java.time LocalDate)))
 
 (set! *warn-on-reflection* true)
@@ -526,41 +526,6 @@
  preferred)))))
 
 
-(def language-reference-sets
- "Defines a mapping between ISO language tags and the language reference sets
- (possibly) installed as part of SNOMED CT. These can be used if a client
- does not wish to specify a reference set (or set of reference sets) to be used
- to derive preferred or acceptable terms, but instead wants some sane defaults
- on the basis of locale as defined by IETF BCP 47."
- {"en-GB" [999001261000000100 ;; NHS realm language (clinical part)
- 999000691000001104 ;; NHS realm language (pharmacy part)
- 900000000000508004 ;; Great Britain English language reference set
- ]
- "en-US" [900000000000509007]})
-
-(defn- select-language-reference-sets
- "Returns the known language reference sets that are in the `installed` set.
- - installed : a set of refset identifiers reference sets."
- [installed]
- (reduce-kv (fn [m k v]
- (if-let [filtered (seq (filter installed v))]
- (assoc m k filtered)
- (dissoc m k)))
- {}
- language-reference-sets))
-
-(defn ordered-language-refsets-from-locale
- "Return an ordered list of language reference sets, as determined by
- the priority list from the user and the set of installed reference sets.
- Parameters
- - priority-list : e.g. en-GB;q=1.0,en-US;q=0.5,fr-FR;q=0.0
- - installed-refsets : e.g. #{900000000000509007 900000000000508004}"
- [^String priority-list installed-refsets]
- (let [installed (select-language-reference-sets installed-refsets)
- priority-list (try (Locale$LanguageRange/parse priority-list) (catch Exception _ []))
- locales (map #(Locale/forLanguageTag %) (keys installed))]
- (mapcat #(get installed %) (map #(.toLanguageTag ^Locale %) (Locale/filter priority-list locales)))))
-
 (defn make-extended-concept [^MapDBStore store concept]
  (let [concept-id (:id concept)
  descriptions (map #(merge % (get-description-refsets store (:id %)))

src/com/eldrix/hermes/import.clj

@@ -1,11 +1,11 @@
 (ns com.eldrix.hermes.import
  "Provides import functionality for processing directories of files"
  (:require
+ [clojure.core.async :as async]
  [clojure.java.io :as io]
+ [clojure.string :as str]
  [clojure.tools.logging :as log]
- [clojure.core.async :as async]
- [com.eldrix.hermes.snomed :as snomed]
- [clojure.string :as str])
+ [com.eldrix.hermes.snomed :as snomed])
  (:import (java.io File)))
 
 (defn is-snomed-file? [f]
@@ -131,8 +131,6 @@
  (async/<!! results-c))))))
 
 (comment
- (require '[clojure.reflect :as reflect])
-
  (snomed/parse-snomed-filename "sct2_Concept_Full_INT_20190731.txt")
  (def filename "/Users/mark/Downloads/uk_sct2cl_30.0.0_20200805000001/SnomedCT_InternationalRF2_PRODUCTION_20190731T120000Z/Snapshot/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_INT_20190731.txt")
  (test-csv filename)