Skip to content
This repository has been archived by the owner on Dec 1, 2023. It is now read-only.

Commit

Permalink
Add a MRSAT table, populate it, use it to populate Bloom filters, and…
Browse files Browse the repository at this point in the history 
… start building a Codesystem/-code framework
  • Loading branch information
Michael O'Keefe committed Jan 6, 2020
1 parent f387ed1 commit 10dda64
Show file tree
Hide file tree
Showing 4 changed files with 165 additions and 19 deletions.
38 changes: 36 additions & 2 deletions create_umls.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,42 @@ fi
echo 'Populating mrrel table'
sqlite3 umls.db ".import MRREL.pipe mrrel"

echo 'Dropping existing mrsat table'
sqlite3 umls.db "drop table if exists mrsat;"

echo 'Creating mrsat table'
sqlite3 umls.db "create table mrsat (
CUI char(8) NOT NULL,
LUI char(8),
SUI char(8),
METAUI varchar(20),
STYPE varchar(50) NOT NULL,
CODE varchar(50),
ATUI varchar(10) NOT NULL,
SATUI varchar(10),
ATN varchar(50),
SAB varchar(20) NOT NULL,
ATV varchar(1000) NOT NULL,
SUPPRESS char(1),
CVF int
);"

# Remove the last pipe from each line, and quote all the fields/escape double quotes
# Because MRSAT has stray unescaped quotes in one of the fields
if [ ! -e MRSAT.pipe ]
then
echo 'Removing last pipe from RRF'
sed 's/|$//' ./resources/terminology/umls_subset/MRSAT.RRF | sed $'s/"/""/g;s/[^|]*/"&"/g' > MRSAT.pipe
fi

echo 'Populating mrsat table'
sqlite3 umls.db ".import MRSAT.pipe mrsat"

echo 'Indexing mrsat(ATN,ATV)'
sqlite3 umls.db "create index idx_at on mrsat(ATN,ATV);"

echo 'Indexing mrconso(tty)'
sqlite3 umls.db "create index idx_tty on mrconso (tty);"
sqlite3 umls.db "create index idx_tty on mrconso(tty);"

echo 'Indexing mrrel(rel,sab)'
sqlite3 umls.db "create index idx_isa on mrrel(REL,SAB);"
Expand All @@ -78,4 +112,4 @@ echo 'Indexing mrconso(aui)'
sqlite3 umls.db "CREATE INDEX idx_aui ON mrconso(AUI);"

echo 'Analyzing Database'
sqlite3 umls.db "ANALYZE;"
sqlite3 umls.db "ANALYZE;"
103 changes: 94 additions & 9 deletions lib/app/endpoint/terminology.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,89 @@ module Inferno
class App
class Terminology < Endpoint
set :prefix, '/fhir'
Inferno::Terminology.register_umls_db('umls.db')
Inferno::Terminology.load_valuesets_from_directory('resources', true)

get '/metadata', provides: ['application/fhir+json', 'application/fhir+xml'] do
capability = FHIR::TerminologyCapabilities.new
capability.id = 'infernoFHIRServer'
capability.url = '/fhir/metadata?mode=terminology'
capability.description = 'TerminologyCapability resource for the Inferno terminology endpoint'
capability.date = Time.now.utc.iso8601
loaded_code_systems = Inferno::Terminology.loaded_code_systems
if params[:mode] == 'terminology'
capability = FHIR::TerminologyCapabilities.new
capability.id = 'InfernoFHIRServer'
capability.url = "#{request.base_url}#{Inferno::BASE_PATH}/fhir/metadata?mode=terminology"
capability.description = 'TerminologyCapability resource for the Inferno terminology endpoint'
capability.date = Time.now.utc.iso8601
capability.status = 'active'
loaded_code_systems = Inferno::Terminology.loaded_code_systems
capability.codeSystem = loaded_code_systems.map { |sys| FHIR::TerminologyCapabilities::CodeSystem.new(uri: sys) }
else
capability = FHIR::CapabilityStatement.new
capability.id = 'InfernoFHIRServer'
capability.url = "#{request.base_url}#{Inferno::BASE_PATH}/fhir/metadata"
capability.description = 'CapabilityStatement resource for the Inferno terminology endpoint'
capability.date = Time.now.utc.iso8601
capability.kind = 'instance'
capability.status = 'active'
capability.fhirVersion = '4.0.1'
capability.rest = FHIR::CapabilityStatement::Rest.new(
mode: 'server',
resource: [
FHIR::CapabilityStatement::Rest::Resource.new(
type: 'ValueSet',
operation: [
FHIR::CapabilityStatement::Rest::Resource::Operation.new(
name: 'validate-code',
definition: 'http://hl7.org/fhir/OperationDefinition/ValueSet-validate-code'
)
]
)
],
operation: [
FHIR::CapabilityStatement::Rest::Resource::Operation.new(
name: 'validate-code',
definition: 'http://hl7.org/fhir/OperationDefinition/Resource-validate'
)
]
)
capability.format = ['xml', 'json']
end
respond_with_type(capability, request.accept, 200)
end

get '/ValueSet/?:id_param?/$validate-code', provides: ['application/fhir+json', 'application/fhir+xml'] do
begin
valueset_validates_code
rescue => e
binding.pry
# TODO: Return an error if the validates code op fails
end
end

post '/ValueSet/?:id_param?/$validate-code', provides: ['application/fhir+json', 'application/fhir+xml'] do
begin
valueset_validates_code
rescue => e
binding.pry
# TODO: Return an error if the validates code op fails
end
end

get '/CodeSystem/?:id_param?/$validate-code', provides: ['application/fhir+json', 'application/fhir+xml'] do
codesystem_validates_code
end

post '/CodeSystem/?:id_param?/$validate-code', provides: ['application/fhir+json', 'application/fhir+xml'] do
begin
parsed_body = FHIR.from_contents(request.body.read)
codesystem_validates_code(parsed_body)
rescue => exception
binding.pry
# TODO: Return an error if the validates code op fails
end
end

def valueset_validates_code
# Get a valueset, in one of the many ways a valueset can be specified
valueset_response = get_valueset(params)
return [400, { 'Content-Type' => 'application/fhir+json' }, valueset_response.to_json] if valueset_response.is_a? FHIR::OperationOutcome
return respond_with_type(valueset_response, request.accept, 400) if valueset_response.is_a? FHIR::OperationOutcome

# now that we have a valueset, let's get the code to validate against it
if params[:code]
Expand All @@ -33,7 +101,7 @@ class Terminology < Endpoint
# TODO: handle this later, because you could have to validate multiple codes...
else
issue = FHIR::OperationOutcome::Issue.new(severity: 'error', code: 'required', details: { text: 'No code parameters were specified to be validated' })
return [400, { 'Content-Type' => 'application/fhir+json' }, FHIR::OperationOutcome.new(issue: issue).to_json]
return respond_with_type(FHIR::OperationOutcome.new(issue: issue), request.accept, 400)
end

code_valid = validate_code(valueset_response, coding, display)
Expand All @@ -49,7 +117,11 @@ class Terminology < Endpoint
]
FHIR::Parameters.new(parameter: params)
end
return [200, { 'Content-Type' => 'application/fhir+json' }, return_params.to_json]
respond_with_type(return_params, request.accept, 200)
end

def codesystem_validates_code(parameters)
# TODO: Implement me
end

private
Expand Down Expand Up @@ -86,6 +158,19 @@ def get_valueset(params)
valueset.process_valueset
valueset
end

def respond_with_type(resource, accept, status = 200)
accept.each do |type|
case type.to_s
when %r{application\/.*json}
return [status, { 'Content-Type' => 'application/fhir+json' }, resource.to_json]
when %r{application\/.*xml}
return [status, { 'Content-Type' => 'application/fhir+xml' }, resource.to_xml]
else
return [status, { 'Content-Type' => 'application/fhir+json' }, resource.to_json]
end
end
end
end
end
end
21 changes: 15 additions & 6 deletions lib/app/utils/terminology.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ def self.create_validators(type)
root_dir = 'resources/terminology/validators/bloom'
FileUtils.mkdir_p(root_dir) unless File.directory?(root_dir)
@known_valuesets.each do |k, vs|
next if (k == 'http://fhir.org/guides/argonaut/ValueSet/argo-codesystem') || (k == 'http://fhir.org/guides/argonaut/ValueSet/languages')
next if (k == 'http://fhir.org/guides/argonaut/ValueSet/argo-codesystem') || (k == 'http://fhir.org/guides/argonaut/ValueSet/languages') || (k == 'http://hl7.org/fhir/us/core/ValueSet/simple-language')

Inferno.logger.debug "Processing #{k}"
filename = "#{root_dir}/#{(URI(vs.url).host + URI(vs.url).path).gsub(%r{[./]}, '_')}.msgpack"
Expand All @@ -51,7 +51,7 @@ def self.create_validators(type)
Inferno::Terminology::Valueset::SAB.each do |k, _v|
Inferno.logger.debug "Processing #{k}"
cs = vs.code_system_set(k)
filename = "#{root_dir}/#{(URI(k).host + URI(k).path).gsub(%r{[./]}, '_')}.msgpack"
filename = "#{root_dir}/#{bloom_file_name(k)}.msgpack"
save_bloom_to_file(cs, filename)
validators << { url: k, file: File.basename(filename), count: cs.length, type: 'bloom' }
end
Expand All @@ -64,15 +64,15 @@ def self.create_validators(type)
next if (k == 'http://fhir.org/guides/argonaut/ValueSet/argo-codesystem') || (k == 'http://fhir.org/guides/argonaut/ValueSet/languages')

Inferno.logger.debug "Processing #{k}"
filename = "#{root_dir}/#{(URI(vs.url).host + URI(vs.url).path).gsub(%r{[./]}, '_')}.csv"
filename = "#{root_dir}/#{bloom_file_name(vs.url)}.csv"
save_csv_to_file(vs.valueset, filename)
validators << { url: k, file: File.basename(filename), count: vs.count, type: 'csv' }
end
vs = Inferno::Terminology::Valueset.new(@db)
Inferno::Terminology::Valueset::SAB.each do |k, _v|
Inferno.logger.debug "Processing #{k}"
cs = vs.code_system_set(k)
filename = "#{root_dir}/#{(URI(k).host + URI(k).path).gsub(%r{[./]}, '_')}.csv"
filename = "#{root_dir}/#{bloom_file_name(k)}.csv"
save_csv_to_file(cs, filename)
validators << { url: k, file: File.basename(filename), count: cs.length, type: 'csv' }
end
Expand Down Expand Up @@ -150,10 +150,19 @@ def self.get_valueset_by_id(id)
@known_valuesets[@valueset_ids[id]] || raise(UnknownValueSetException, id)
end

def self.bloom_file_name(codesystem)
uri = URI(codesystem)
if uri.host && uri.port
return (uri.host + uri.path).gsub(%r{[./]}, '_')
else
return codesystem.gsub(%r{[.\W]}, '_')
end
end

def self.loaded_code_systems
@loaded_code_systems ||= @known_valuesets.flat_map do |_, vs|
vs.valueset.collect {|s| s[:system] }.uniq
end.uniq
vs.included_code_systems.uniq
end.uniq.compact
end

class UnknownValueSetException < StandardError
Expand Down
22 changes: 20 additions & 2 deletions lib/app/utils/valueset.rb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ class Valueset
'http://loinc.org' => 'LNC',
'http://snomed.info/sct' => 'SNOMEDCT_US',
'http://www.icd10data.com/icd10pcs' => 'ICD10CM',
'http://hl7.org/fhir/sid/icd-10-cm' => 'ICD10CM',
'http://hl7.org/fhir/sid/icd-9-cm' => 'ICD9CM',
'http://unitsofmeasure.org' => 'NCI_UCUM',
'http://hl7.org/fhir/ndfrt' => 'NDFRT',
'http://nucc.org/provider-taxonomy' => 'NUCCPT',
Expand All @@ -45,6 +47,8 @@ class Valueset

# https://www.nlm.nih.gov/research/umls/knowledge_sources/metathesaurus/release/attribute_names.html
FILTER_PROP = {
'CLASSTYPE' => 'LCN',
'DOC' => 'Doc',
'SCALE_TYP' => 'LOINC_SCALE_TYP'
}.freeze

Expand Down Expand Up @@ -78,6 +82,10 @@ def count
@valueset.length
end

def included_code_systems
@valueset_model.compose.include.map(&:system).compact.uniq
end

# Creates the whole valueset
#
# Creates a [Set] representing the valueset
Expand Down Expand Up @@ -219,8 +227,14 @@ def filter_code_set(system, filter = nil, _version = nil)
filtered_set.add(system: system, code: row[0])
end
elsif ['=', 'in', nil].include? filter&.op
@db.execute("SELECT code FROM mrconso WHERE SAB = '#{SAB[system]}' AND #{filter_clause.call(filter)}") do |row|
filtered_set.add(system: system, code: row[0])
if FILTER_PROP[filter.property]
@db.execute("SELECT code FROM mrsat WHERE ATN = '#{filter_prop_or_self(filter.property)}' AND ATV = '#{filter_prop_or_self(filter.value)}'") do |row|
filtered_set.add(system: system, code: row[0])
end
else
@db.execute("SELECT code FROM mrconso WHERE SAB = '#{SAB[system]}' AND #{filter_clause.call(filter)}") do |row|
filtered_set.add(system: system, code: row[0])
end
end
elsif filter&.op == 'is-a'
filtered_set = filter_is_a(system, filter)
Expand Down Expand Up @@ -289,6 +303,10 @@ def filter_is_a(system, filter)
desired_children
end

def filter_prop_or_self(prop)
FILTER_PROP[prop] || prop
end

class FilterOperationException < StandardError
def initialize(filter_op)
super("Cannot Handle Filter Operation: #{filter_op}")
Expand Down

0 comments on commit 10dda64

Please sign in to comment.