xquery version "3.1";
(:
    ART-DECOR® STANDARD COPYRIGHT AND LICENSE NOTE
    Copyright © ART-DECOR Expert Group and ART-DECOR Open Tools GmbH
    see https://docs.art-decor.org/copyright and https://docs.art-decor.org/licenses

    This file is part of the ART-DECOR® tools suite.
:)
(: TODO: Should come up with some kind of marker in content retrieved from a repo just to make it clear that is not actual local to this file? :)
module namespace comp = "http://art-decor.org/ns/art-decor-compile";

import module namespace get      = "http://art-decor.org/ns/art-decor-settings" at "../modules/art-decor-settings.xqm";
import module namespace art      = "http://art-decor.org/ns/art" at "../modules/art-decor.xqm";
import module namespace vs       = "http://art-decor.org/ns/decor/valueset" at "api-decor-valueset.xqm";
import module namespace iss      = "http://art-decor.org/ns/decor/issue" at "api-decor-issue.xqm";
import module namespace templ    = "http://art-decor.org/ns/decor/template" at "api-decor-template.xqm";
import module namespace adserver = "http://art-decor.org/ns/art-decor-server" at "api-server-settings.xqm";
import module namespace utilmp   = "http://art-decor.org/ns/api/util-conceptmap" at "../../api/modules/library/util-conceptmap-lib.xqm";
declare namespace xdb            = "http://exist-db.org/xquery/xmldb";
declare namespace request        = "http://exist-db.org/xquery/request";
declare namespace response       = "http://exist-db.org/xquery/response";
declare namespace datetime       = "http://exist-db.org/xquery/datetime";
declare namespace error          = "http://art-decor.org/ns/decor/compile/error";
declare option exist:serialize "method=xml media-type=text/xml";

declare variable $comp:debug                := false();
declare variable $comp:strArtURL            := adserver:getServerURLArt();
declare variable $comp:strArt3URL           := adserver:getServerURLArt3();
declare variable $comp:strDecorServicesURL  := adserver:getServerURLServices();
declare variable $comp:strFhirServicesURL   := adserver:getServerURLFhirServices();

(: set output size limit to 9M nodes (XML fragments), default is 1M nodes :)
declare option exist:output-size-limit "9000000";

(: Filtered for language :)
declare %private function comp:compileProject($node as element(project), $language as xs:string) as element() {
<project>
{
    $node/@*,
    if ($language = '*') then (
        $node/name,
        $node/desc
    ) else (
        $node/name[@language = $language][1],
        $node/desc[@language = $language][1]
    ),
    $node/copyright,
    $node/license,
    $node/author,
    $node/reference,
    $node/restURI,
    $node/defaultElementNamespace,
    $node/contact,
    $node/buildingBlockRepository,
    for $versionRelease in ($node/version | $node/release)
    order by $versionRelease/@date descending
    return
        element {name($versionRelease)} {
            $versionRelease/@*,
            if ($language = '*') then ($versionRelease/note | $versionRelease/desc) else (
                if ($versionRelease[note]) then
                    if ($versionRelease/note[@language = $language]) then $versionRelease/note[@language = $language][1] else $versionRelease/note[1]
                else (
                    if ($versionRelease/desc[@language = $language]) then $versionRelease/desc[@language = $language][1] else $versionRelease/desc[1]
                )
            )
        }
}
</project>
};

(: Calls central function getFullDatasetTree :)
declare %private function comp:compileDatasets($node as element(datasets)?, $compiledScenarios as element(scenarios)?, $language as xs:string, $filters as element(filters)?) as element() {
    (: transactions may point to datasets in other projects. These would not be covered necessarily by a dataset/@ref in the $node.
        only transactions we actually need to do should be in the compiledScenarios set, so we depend on the accuracy there.
    :)
    let $transactionDatasets    := 
        for $representingTemplate in $compiledScenarios//representingTemplate[@sourceDataset[not(. = '')]]
        let $dsid   := $representingTemplate/@sourceDataset
        let $dsed   := $representingTemplate/@sourceDatasetFlexibility
        group by $dsid, $dsed
        return (
            if ($node/dataset[@id = string($dsid)][@effectiveDate = string($dsed)]) then () else (
                let $dataset    := art:getDataset($dsid, $dsed)
                let $sdsid      := $dataset/@id
                let $sdsed      := $dataset/@effectiveDate
                return
                    if ($node/dataset[@id = $sdsid][@effectiveDate = $sdsed]) then (
                        (: already have it in our project :)
                    ) 
                    else (
                        (: add to our total set :)
                        $dataset
                    )
            )
        )
    
    (: add here for comp:compileIds() :)
    let $transactionIds         := $transactionDatasets/ancestor::decor/ids
    
    let $nodes                  := 
        for $child in $node/node()
        return
            if ($child[name() = 'dataset']) then ( 
                let $dsid   := $child/@id
                let $dsed   := $child/@effectiveDate
                                                                (:if there's no */@ref in $filters, assume no filtering is requested:)
                return
                if (empty($filters) or $filters/dataset[@ref = $dsid][@flexibility = $dsed]) then 
                    $child
                else (
                    (:filtered out...:)
                    comment {concat($child/name(),' was filtered id=''',$dsid,''' effectiveDate=''', $dsed, ''' ', $child/name[1]/replace(., '--', '-'), ' ')}
                )
            )
            else (
                $child
            )
    let $dsmap                  := map:merge(
        for $ds in $nodes | $transactionDatasets
        let $dsid   := $ds/@id
        let $dsed   := $ds/@effectiveDate
        group by $dsid, $dsed
        return 
            if ($ds[@id]) then map:entry(concat($dsid, $dsed), ()) else ()
    )
    let $fullDatasets           := comp:getContainedConcepts($nodes | $transactionDatasets, $dsmap)
    
    (: build map of oid names. is more costly when you do that deep down :)
    let $oidnamemap             :=
        map:merge(
            for $oid in distinct-values($fullDatasets//@id | $fullDatasets//@ref)
            return
                map:entry($oid, art:getNameForOID($oid, $language, ()))
        )
    return
    <datasets>
    {
        $node/@*                                                   (:doesn't currently have any, but who knows... better not to loose data:)
        ,
        for $ds in $fullDatasets
        let $dsid   := $ds/@id
        let $dsed   := $ds/@effectiveDate
        group by $dsid, $dsed
        order by $dsid, $dsed
        return 
            if ($ds[@id]) then art:getFullDatasetTree($ds[1], (), (), $language, (), true(), $oidnamemap) else $ds
        ,
        (: add here for comp:compileIds() :)
        $transactionIds
    }
    </datasets>
};

declare %private function comp:getContainedConcepts($nodes as item()*, $dsmap as map(*)?) as item()* {
    let $datasets         :=
        for $c in $nodes//contains[not(ancestor::history)]
        let $deid   := $c/@ref
        let $deed   := $c/@flexibility
        group by $deid, $deed
        return (
            let $ds := art:getConcept($deid, $deed)/ancestor::dataset
            return
            if (empty($ds)) then
                comment { ' ', for $cc in $c return concat('concept id=', $cc/../@id, ' contains unresolved id=', $deid, ' flexibility=', $deed, ' '), ' '}
            else
            if (map:contains($dsmap, concat($ds/@id, $ds/@effectiveDate))) then () else (
                $ds
            )
        )
    let $datasetsInherit  :=
        for $c in $nodes//inherit[not(ancestor::history | concept)]
        let $deid   := $c/@ref
        let $deed   := $c/@effectiveDate
        group by $deid, $deed
        return (
            let $targetConcept  := art:getConcept($deid, $deed)
            let $ccid           := $targetConcept/contains[1]/@ref
            let $cced           := $targetConcept/contains[1]/@flexibility
            return
                if ($targetConcept[contains]) then (
                    let $ds := art:getConcept($ccid, $cced)/ancestor::dataset
                    return
                    if (empty($ds)) then
                        comment { ' ', for $cc in $c return concat('concept id=', $cc/../@id, ' contains unresolved id=', $deid, ' flexibility=', $deed, ' '), ' '}
                    else
                    if (map:contains($dsmap, concat($ds/@id, $ds/@effectiveDate))) then () else
                    if ($datasets[@id = $ds/@id]/@effectiveDate = $ds/@effectiveDate) then ()
                    else (
                        $ds
                    )
                )
                else ()
        )
    
    let $datasets       := $datasets | $datasetsInherit
    let $dsmap          := map:merge((
        $dsmap,
        for $ds in $datasets
        let $dsid   := $ds/@id
        let $dsed   := $ds/@effectiveDate
        group by $dsid, $dsed
        return 
            if ($ds[@id]) then map:entry(concat($dsid, $dsed), ()) else ()
    ))
    
    return (
        if ($datasets//contains[not(ancestor::history)]) then (
            $nodes | comp:getContainedConcepts($datasets, $dsmap)
        )
        else (
            $nodes | $datasets
        )
    )
};

(:recursively handle the tree:)
declare %private function comp:compileScenarios($node as element()?, $filters as element(filters)?, $runtimeonly as xs:boolean) as node()* {
    if (not(comp:isCompilationFilterActive($filters))) then (
        (:no filtering necessary. return as-is:)
        $node
    )
    else 
    if ($node[name() = 'scenarios']) then (
        <scenarios>
        {
            $node/@*,
            for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
        }
        </scenarios>
    )
    else 
    if ($node[name() = 'scenario']) then (
        let $scid   := $node/@id
        let $sced   := $node/@effectiveDate
        
        return
        (: copy only scenarios which are listed in filters :)
        if (comp:isCompilationFilterActive($filters)) then
            if ($filters/scenario[@ref = $scid][@flexibility = $sced]) then
                element {$node/name()} {
                    $node/@*,
                    for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
                }
            else (
                (:filtered out...:)
                comment {concat($node/name(),' was filtered id=''',$scid,''' effectiveDate=''', $sced, ''' ', $node/name[1]/replace(., '--', '-'),' ')}
            )
        else
        if ($runtimeonly) then
            if ($node/descendant-or-self::transaction/representingTemplate/@ref) then
                element {$node/name()} {
                    $node/@*,
                    for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
                }
            else (
                (:filtered out...:)
                comment {concat($node/name(),' was filtered id=''',$scid,''' effectiveDate=''', $sced, ''' ', $node/name[1]/replace(., '--', '-'),' ')}
            )
        else (
            element {$node/name()} {
                $node/@*,
                for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
            }
        )
    )
    else 
    if ($node[name() = 'transaction']) then (
        let $trid   := $node/@id
        let $tred   := $node/@effectiveDate
        
        return
        if (comp:isCompilationFilterActive($filters)) then
            if ($filters/transaction[@ref = $trid][@flexibility = $tred]) then
                element {$node/name()} {
                    $node/@*,
                    for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
                }
            else (
                (:filtered out...:)
                comment {concat($node/name(),' (',$node/@type,') was filtered id=''',$trid,''' effectiveDate=''', $tred, ''' ', $node/name[1]/replace(., '--', '-'),' ')}
            )
        else
        if ($runtimeonly) then
            if ($node/descendant-or-self::transaction/representingTemplate/@ref) then
                element {$node/name()} {
                    $node/@*,
                    for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
                }
            else (
                (:filtered out...:)
                comment {concat($node/name(),' (',$node/@type,') was filtered id=''',$trid,''' effectiveDate=''', $tred, ''' ', $node/name[1]/replace(., '--', '-'),' ')}
            )
        else (
            element {$node/name()} {
                $node/@*,
                for $child in $node/* return comp:compileScenarios($child, $filters, $runtimeonly)
            }
        )
    )
    else 
    if ($node[name() = 'representingTemplate'] and $runtimeonly) then
        element {$node/name()} {
            $node/(@* except (@sourceDataset | @sourceDatasetFlexibility))
        }
    else (
        (: copy any other node :)
        $node
    )
};

declare %private function comp:compileTerminology($node as element(terminology), $filters as element(filters)?, $runtimeonly as xs:boolean) as element(terminology) {
    let $filterActive       := $filters/@filteringActive = 'true'
    let $projectPrefix      := $node/ancestor::decor/project/@prefix/string()
    (: Two situations
       1. Filtering is off: pick everything in the project + what's in filters as the list in filters 
          is harvested from the full expanded template chain. When there's template[@ref], then it is 
          very likely that there's also vocabulary unaccounted for in valueSet[@ref]
       2. Filtering is on: work based on the compiled list only. The list is expected to contain 
          reference to each and any relevant valueSet
    :)
    let $valueSetsInScope   := 
        if ($filterActive) then
            $filters/valueSet
        else (
            $node/valueSet | $filters/valueSet
        )
    
    let $conceptMapsInScope :=
        for $cm in $node/conceptMap | $filters/conceptMap
        let $cmid := $cm/@id | $cm/@ref
        let $cmed := $cm/@effectiveDate
        group by $cmid, $cmed
        return
            if ($cm/@id) then $cm[1] else (
                ($cm[@ref])[1],
                for $cmchild in utilmp:getConceptMap($projectPrefix, (), (), $cmid, (), false(), true())[@id]
                return
                    <conceptMap>
                    {
                        $cmchild/@*,
                        $cmchild/(* except issueAssociation)
                    }
                    </conceptMap>
            )
    let $conceptMapsInScope :=
        if ($filterActive) then
            $conceptMapsInScope[sourceScope[@ref = $valueSetsInScope/@ref] | targetScope[@ref = $valueSetsInScope/@ref]]
        else (
            $conceptMapsInScope
        )
    
    (: It should not happen, but if ConceptMaps exist that reference ValueSets that are not referenced in the 
       terminology section we should bring them in scope for compilation here :)
    let $valueSetsInScope   := (
        $valueSetsInScope, for $vs in $conceptMapsInScope/sourceScope | $conceptMapsInScope/targetScope return <valueSet>{$vs/@*}</valueSet>
    )
    
    (: Filter list may contain both dynamic and static where static == the newest. 
       This would lead to duplicates, so build first and group second:)
    let $compiledValueSets  :=
        for $valueSet in $valueSetsInScope
        let $vsid       := $valueSet/@id | $valueSet/@ref
        let $vsed       := $valueSet/@effectiveDate | $valueSet/@flexibility
        let $vsurl      := $valueSet/@url[not(. = '')]
        let $vsident    := $valueSet/@ident[not(. = '')]
        group by $vsid, $vsed, $vsurl, $vsident
        return (
            let $expandedValueSets  := 
                if ($valueSet[@id]) then
                    vs:getExpandedValueSet(($valueSet[@id])[1], ($vsident, $projectPrefix)[1], (), (), false())
                else (
                (: check the project it was referred from, but if that is a cached project, this will not hold. Check from original project next if empty :)
                    if (empty($vsident)) then
                        vs:getExpandedValueSetByRef($vsid, if (empty($vsed)) then 'dynamic' else $vsed, $projectPrefix, (), (), false())//valueSet[@id]
                    else (
                        let $vss := vs:getExpandedValueSetByRef($vsid, if (empty($vsed)) then 'dynamic' else $vsed, $vsident, (), (), false())//valueSet[@id]
                        return
                            if ($vss) then $vss else (
                                vs:getExpandedValueSetByRef($vsid, if (empty($vsed)) then 'dynamic' else $vsed, $projectPrefix, (), (), false())//valueSet[@id]
                            )
                    )
                )
            for $expandedValueSet in $expandedValueSets
            return
                <valueSet>
                {
                    $expandedValueSet/@*,
                    $expandedValueSet/parent::repository/(@ident, @url, @referencedFrom),
                    for $n in $expandedValueSet/desc 
                    return 
                        art:parseNode($n)
                    ,
                    $expandedValueSet/sourceCodeSystem,
                    $expandedValueSet/node()[not(self::desc | self::sourceCodeSystem | self::conceptList)],
                    if ($expandedValueSet[conceptList]) then (
                        <conceptList>
                        {
                            for $c in $expandedValueSet/conceptList/*
                            (: rewrite concept with NullFlavor to exception. NullFlavor never is a concept :)
                            let $cname  := 
                                if ($c[self::concept][@codeSystem = '2.16.840.1.113883.5.1008']) then 
                                    'exception'
                                else (name($c))
                            return
                                element {$cname} {
                                    $c/(@* except @exception),
                                    (: rewrite/add @exception of includes for the HL7 ValueSet for 
                                    NullFlavor to exception="true". NullFlavor never is a concept :)
                                    if ($c[self::include][@ref = '2.16.840.1.113883.1.11.10609'][not(@exception = 'true')]) then
                                        attribute exception {'true'}
                                    else ($c/@exception)
                                    ,
                                    for $node in $c/node()
                                    return
                                        if ($node instance of element(desc)) then art:parseNode($node) else $node
                                }
                        }
                        </conceptList>
                    ) else ()
                }
                </valueSet>
        )
    
    let $compiledConceptMaps  :=
        for $cm in $conceptMapsInScope
        let $cmid := $cm/@id | $cm/@ref
        let $cmed := $cm/@effectiveDate | $cm/@flexibility
        group by $cmid, $cmed
        order by replace(replace(concat($cmid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
        return
            if ($cm/@id) then $cm[@id][1] else (
                $cm[1],
                for $cmchild in utilmp:getConceptMap($projectPrefix, (), (), $cmid, (), false(), true())[@id]
                return
                    <conceptMap>
                    {
                        $cmchild/@*,
                        $cmchild/(* except issueAssociation)
                    }
                    </conceptMap>
            )
    
    return
        <terminology>
        {
            (: when datasets are compiled, they internalize all terminologyAssociations. when we filter we can't resolve all terminologyAssociations anyway.
                so... do we even need all these terminologyAssociations?
            :)
            if ($runtimeonly) then () else ((:$node/terminologyAssociation:))
            ,
            if ($runtimeonly) then () else $node/codeSystem
            ,
            for $valueSet in $compiledValueSets
            let $vsid           := $valueSet/@id | $valueSet/@ref
            let $vsed           := $valueSet/@effectiveDate
            group by $vsid, $vsed
            order by $vsid, $vsed
            return $valueSet[1]
            ,
            if (string($comp:debug)='true') then (
                for $valueSet in $node/valueSet[not(concat(@id,@effectiveDate)=$compiledValueSets/concat(@id,@effectiveDate))][not(@ref=$compiledValueSets/@id)]
                let $vsid := $valueSet/@id | $valueSet/@ref
                let $vsed := $valueSet/@effectiveDate | $valueSet/@flexibility
                group by $vsid, $vsed
                order by replace(replace(concat($vsid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
                return (
                    comment {'Not adding valueSet', for $att in $valueSet[1]/(@id, @ref, @name, @displayName, @effectiveDate, @flexibility) return name($att) || '="' || $att || '"'},'&#10;'
                )
                ,
                for $valueSet in $compiledValueSets[not(concat(@id,@effectiveDate)=$node/valueSet/concat(@id,@effectiveDate))][not(@id=$node/valueSet/@ref)]
                let $vsid := $valueSet/@id | $valueSet/@ref
                let $vsed := $valueSet/@effectiveDate | $valueSet/@flexibility
                group by $vsid, $vsed
                order by replace(replace(concat($vsid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
                return (
                    comment {'Adding valueSet', for $att in $valueSet[1]/(@id, @ref, @name, @displayName, @effectiveDate, @flexibility) return name($att) || '="' || $att || '"'},'&#10;'
                )
            ) else ()
            ,
            for $conceptMap in $conceptMapsInScope
            let $cmid := $conceptMap/@id | $conceptMap/@ref
            let $cmed := $conceptMap/@effectiveDate
            group by $cmid, $cmed
            order by replace(replace(concat($cmid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
            return
                $conceptMap[1]
            ,
            if (string($comp:debug)='true') then (
                for $conceptMap in $node/conceptMap[not(concat(@id,@effectiveDate)=$compiledConceptMaps/concat(@id,@effectiveDate))][not(@ref=$compiledConceptMaps/@id)]
                let $cmid := $conceptMap/@id | $conceptMap/@ref
                let $cmed := $conceptMap/@effectiveDate | $conceptMap/@flexibility
                group by $cmid, $cmed
                order by replace(replace(concat($cmid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
                return (
                    comment {'Not adding conceptMap', for $att in $conceptMap[1]/(@id, @ref, @displayName, @effectiveDate, @flexibility) return name($att) || '="' || $att || '"'},'&#10;'
                )
                ,
                for $conceptMap in $compiledConceptMaps[not(concat(@id,@effectiveDate)=$node/conceptMap/concat(@id,@effectiveDate))][not(@id=$node/conceptMap/@ref)]
                let $cmid := $conceptMap/@id | $conceptMap/@ref
                let $cmed := $conceptMap/@effectiveDate | $conceptMap/@flexibility
                group by $cmid, $cmed
                order by replace(replace(concat($cmid, '.'), '\.', '.0000000000'), '.0*([0-9]{9,})', '.$1')
                return (
                    comment {'Adding conceptMap', for $att in $conceptMap[1]/(@id, @ref, @displayName, @effectiveDate, @flexibility) return name($att) || '="' || $att || '"'},'&#10;'
                )
            ) else ()
        }
        </terminology>
};

(: additionalNodes are elements found through compilation of references valueSets/templates and the like :)
declare %private function comp:compileIds($node as element(ids), $otherIds as element()*, $additionalNodes as element()*, $runtimeonly as xs:boolean) as element(ids) {
if ($runtimeonly) then (
    <ids>
    {
        $node/baseId,
        $additionalNodes[self::ids]/baseId,
        $node/defaultBaseId
    }
    </ids>
) else (
    let $projectPrefix      := $node/ancestor::decor/project/@prefix/string()
    let $projectLanguage    := $node/ancestor::decor/project/@defaultLanguage/string()
    let $allDefinedOIDs     := $node/id/@root | $otherIds/id/@root
    let $allOIDs            := ($node/ancestor::decor//@codeSystem[not(ancestor::example | ancestor::ids)] | 
                                $node/ancestor::decor//@root[not(ancestor::example | ancestor::ids)] |
                                $node/ancestor::decor//identifierAssociation/@ref[not(ancestor::example)] |
                                $additionalNodes//@codeSystem[not(ancestor::example | ancestor::ids)] | 
                                $additionalNodes//@root[not(ancestor::example | ancestor::ids)] |
                                $additionalNodes//identifierAssociation/@ref[not(ancestor::example)])
    let $allDeclaredOIDs    := 
        for $oidString in $allOIDs
        for $oid in tokenize($oidString, '\|')
        return
            normalize-space($oid)
    
    let $otherBaseIds       :=
        for $baseId in $otherIds/baseId
        let $ref    := $baseId/@id
        group by $ref
        order by $ref
        return if ($node/baseId[@id = $ref]) then () else $baseId[1]
    let $otherIdRoots1      :=
        for $baseId in $otherIds/id
        let $ref    := $baseId/@root
        group by $ref
        order by $ref
        return if ($node/id[@root = $ref]) then () else $baseId[1]
    let $otherIdRoots2      :=
        for $oid in distinct-values($allDeclaredOIDs)[not(. = $allDefinedOIDs)]
        (:  these aren't covered by art:getNameForOID. They are inexpensive to get, 
            while art:getNameForOID is extremely expensive :)
        let $local          := 
            for $l in ($get:colDecorData//template[@id = $oid] | 
                        $get:colDecorData//template[@ref = $oid] |
                        $get:colDecorData//codeSystem[@id = $oid] | 
                        $get:colDecorData//codeSystem[@ref = $oid] |
                        $get:colDecorCache//template[@id = $oid][ancestor::decor] | 
                        $get:colDecorCache//template[@ref = $oid][ancestor::decor] |
                        $get:colDecorCache//codeSystem[@id = $oid][ancestor::decor] | 
                        $get:colDecorCache//codeSystem[@ref = $oid][ancestor::decor])
            order by $l/@effectiveDate descending 
            return $l
        let $oidName        := ($local/@displayName, $local/@name)[1]
        let $oidName        := if (empty($oidName)) then art:getNameForOID($oid, $projectLanguage, ()) else $oidName
        let $hl7v2table0396 := art:getHL7v2Table0396MnemonicForOID($oid, (), ())
        return
            if (string-length($oidName) gt 0) then
                <id root="{$oid}">
                    <designation displayName="{$oidName}" language="{$projectLanguage}">{data($oidName)}</designation>
                {
                    if ($hl7v2table0396) then <property name="{$get:strKeyHL7v2Table0396CodePrefd}">{$hl7v2table0396}</property> else ()
                }
                </id>
            else (
                comment {'Could not find OID name for', string-join(distinct-values($allOIDs[. = $oid]/concat(ancestor::decor/project/@prefix, ' (', name(),')')), ' / '),$oid}
            )
    return
        <ids>
        {
            $node/baseId,
            if (empty($otherBaseIds)) then () else (
                '&#10;        ',
                comment {'BEGIN Base IDs added through compilation'}
                ,
                $otherBaseIds
            )
            ,
            $node/defaultBaseId,
            $node/id,
            if (empty($otherIdRoots1) and empty($otherIdRoots2)) then () else (
                '&#10;        ',
                comment {'BEGIN IDs added through compilation'},
                for $baseId in $otherIdRoots1 | $otherIdRoots2[self::id]
                let $ref    := $baseId/@root
                order by $ref
                return $baseId
                ,
                for $comment in $otherIdRoots2[self::comment]
                return (
                    '&#10;        ', $comment
                )
            )
            (: when datasets are compiled, they internalize all identifierAssociations. when we filter we can't resolve all identifierAssociations anyway.
                so... do we even need all these identifierAssociations?
            :)
            (:,
            $node/identifierAssociation:)
        }
        </ids>
)
};

declare %private function comp:compileRules($node as element(rules)?, $filters as element(filters)?, $runtimeonly as xs:boolean) as element(rules)? {
let $projectPrefix  := $node/ancestor::decor/project/@prefix

return
    if ($node) then
        <rules>
        {
            let $startTemplates     :=
                if ($filters) then (
                    for $template in $filters/template
                    let $tmid       := $template/@ref
                    let $tmed       := $template/@flexibility[not(. = 'dynamic')]
                    group by $tmid, $tmed
                    return
                        templ:getTemplateById($tmid, $tmed, $projectPrefix)/template/template[@id]
                ) 
                else (
                    $node/template[@id]
                    ,
                    for $template in $node/template[@ref]
                    let $tmid       := $template/@ref
                    let $tmed       := $template/@flexibility[not(. = 'dynamic')]
                    group by $tmid, $tmed
                    return
                        templ:getTemplateById($tmid, $tmed, $projectPrefix)/template/template[@id]
                )
            
            (:get everything hanging off from the current template so we don't miss anything, but may have duplicates in the result:)
            let $templateChain          := $startTemplates | 
                                            templ:getTemplateChain($projectPrefix, $startTemplates, 
                                                map:merge(for $starttemplate in $startTemplates return map:entry(concat($starttemplate/@id, $starttemplate/@effectiveDate), ''))
                                            )
            
            for $templates in $templateChain
            let $tmid       := ($templates/@id, $templates/@ref)
            let $tmed       := $templates/@effectiveDate
            group by $tmid, $tmed
            order by $tmid, $tmed descending
            return (
                '&#10;        ',
                comment {' ', ($templates/@displayName, $templates/@name)[1]/replace(., '--', '-'), ' '},
                '&#10;        ',
                let $templateAssociations     := 
                    if ($runtimeonly) then () else (
                        $get:colDecorData//templateAssociation[@templateId = $tmid][@effectiveDate = $tmed] | 
                        $get:colDecorCache//templateAssociation[@templateId = $tmid][@effectiveDate = $tmed]
                    )
                return
                    if ($templateAssociations[concept]) then
                        <templateAssociation templateId="{$tmid}" effectiveDate="{$tmed}">
                        {
                            for $concept in $templateAssociations/concept
                            let $elid   := $concept/@elementId | $concept/@elementPath
                            let $deid   := $concept/@ref
                            let $deed   := $concept/@effectiveDate
                            group by $deid, $deed, $elid
                            order by $deid, $deed, $elid
                            return (
                                let $currentProject     := $concept[ancestor::decor/project[@prefix = $projectPrefix]]
                                return
                                    if ($currentProject) then $currentProject[1] else (
                                        <concept>
                                        {
                                            $concept[1]/(@* except (@url, @ident)), 
                                            attribute url {($concept[1]/ancestor::decor/@deeplinkprefixservices, $comp:strDecorServicesURL)[1]},
                                            attribute ident {$concept[1]/ancestor::decor/project/@prefix}
                                        }
                                        </concept>
                                    )
                            )
                        }
                        </templateAssociation>
                    else ()
                ,
                ($templates[@id], $templates)[1]
            )
        }
        {
            let $startObjects       :=
                if ($filters) then (
                    for $template in $filters/questionnaire
                    let $tmid       := $template/@ref
                    let $tmed       := $template/@flexibility[not(. = 'dynamic')]
                    group by $tmid, $tmed
                    return (
                        if ($tmed castable as xs:dateTime) then 
                            $get:colDecorData//questionnaire[@id = $tmid][@effectiveDate = $tmed]
                        else (
                            let $q  := $get:colDecorData//questionnaire[@id = $tmid]
                            return
                                $q[@effectiveDate = string(max($q/xs:dateTime(@effectiveDate)))]
                        )
                    )
                ) 
                else (
                    $node/questionnaire[@id]
                    ,
                    for $template in $node/questionnaire[@ref]
                    let $tmid       := $template/@ref
                    let $tmed       := $template/@flexibility[not(. = 'dynamic')]
                    group by $tmid, $tmed
                    return
                        if ($tmed castable as xs:dateTime) then 
                            $get:colDecorData//questionnaire[@id = $tmid][@effectiveDate = $tmed]
                        else (
                            let $q  := $get:colDecorData//questionnaire[@id = $tmid]
                            return
                                $q[@effectiveDate = string(max($q/xs:dateTime(@effectiveDate)))]
                        )
                )
            
            for $questionnaires in $startObjects
            let $tmid       := ($questionnaires/@id, $questionnaires/@ref)
            let $tmed       := $questionnaires/@effectiveDate
            group by $tmid, $tmed
            order by $tmid, $tmed descending
            return (
                '&#10;        ',
                comment {' ', $questionnaires/name[1]/replace(., '--', '-'), ' '},
                '&#10;        ',
                let $associations    := 
                    if ($runtimeonly) then () else (
                        $get:colDecorData//questionnaireAssociation[@questionnaireId = $tmid][@questionnaireEffectiveDate = $tmed] | 
                        $get:colDecorCache//questionnaireAssociation[@questionnaireId = $tmid][@questionnaireEffectiveDate = $tmed]
                    )
                return
                    if ($associations[concept]) then
                        <questionnaireAssociation questionnaireId="{$tmid}" questionnaireEffectiveDate="{$tmed}">
                        {
                            for $concept in $associations/concept
                            let $elid   := $concept/@elementId | $concept/@elementPath
                            let $deid   := $concept/@ref
                            let $deed   := $concept/@effectiveDate
                            group by $deid, $deed, $elid
                            order by $deid, $deed, $elid
                            return (
                                let $currentProject     := $concept[ancestor::decor/project[@prefix = $projectPrefix]]
                                return
                                    if ($currentProject) then $currentProject[1] else (
                                        <concept>
                                        {
                                            $concept[1]/(@* except (@url, @ident)), 
                                            attribute url {($concept[1]/ancestor::decor/@deeplinkprefixservices, $comp:strDecorServicesURL)[1]},
                                            attribute ident {$concept[1]/ancestor::decor/project/@prefix}
                                        }
                                        </concept>
                                    )
                            )
                        }
                        </questionnaireAssociation>
                    else ()
                ,
                ($questionnaires[@id], $questionnaires)[1]
            )
        }
        </rules>
    else (
        <rules/>
    )
};

declare %private function comp:compileIssues($node as element(issues)?, $language as xs:string, $filters as element(filters)?) as element(issues)? {
let $projectPrefix  := $node/ancestor::decor/project/@prefix

return
    <issues>
    {
        attribute notifier {if ($node[@notifier]) then $node/@notifier else 'on'},
        $node/(@* except @notifier)
    }
    {
        for $childnode in $node/*
        return
            if ($childnode/name() = 'issue') then
                <issue>
                {
                    $childnode/@*,
                    $childnode/object,
                    for $eventnode in $childnode/tracking | $childnode/assignment
                    order by $eventnode/@effectiveDate
                    return
                        $eventnode
                    ,
                    $childnode/(* except (object | assignment | tracking))
                }
                </issue>
            else (
                $childnode
            )
    }
    </issues>
};

declare function comp:compileDecor($decorproject as element(decor), $language as xs:string) as element(decor) {
    let $now := substring(string(current-dateTime()),1,19)
    return comp:compileDecor($decorproject, $language, $now, (), false(), false())
};

declare function comp:compileDecor($decorproject as element(decor), $language as xs:string, $now as xs:string) as element(decor) {
    comp:compileDecor($decorproject, $language, $now, (), false(), false())
};

declare function comp:compileDecor($decorproject as element(decor), $language as xs:string, $now as xs:string, $filters as element(filters)?) as element(decor) {
    comp:compileDecor($decorproject, $language, $now, $filters, false(), false())
};

declare function comp:compileDecor($decorproject as element(decor), $language as xs:string, $now as xs:string, $filters as element(filters)?, $testfilters as xs:boolean) as element() {
    comp:compileDecor($decorproject, $language, $now, $filters, $testfilters, false())
};

(:~ Create a compilation

    @param $decor           - The DECOR project to compile
    @param $language        - The project language to compile for or '*' for all, if empty uses project default language
    @param $now             - The timestamp to add to the compilation, if empty uses current-dateTime()
    @param $filters         - The filters to apply if any. Note: pre-process any filters through comp:getFinalCompilationFilters($decor, $filters) before sending them here
    @param $testfilters     - Boolean value that determines if this function returns the fully processed filters or the DECOR compilation itself. Fully processed filters also contain terminology
    @param $runtimeonly     - Boolean value that guides the compilation for runtime usage or not. Runtime compilations don't have any datasets and carry only transactions that point to a template
    @return Compiled DECOR project or fully compiled filters or nothing
:)
declare function comp:compileDecor($decorproject as element(decor), $language as xs:string, $now as xs:string, $filters as element(filters)?, $testfilters as xs:boolean, $runtimeonly as xs:boolean) as element() {
    let $language               := if ($decorproject/project/name[@language = $language] or $language = '*') then $language else $decorproject/project/@defaultLanguage
    let $filtersActive          := comp:isCompilationFilterActive($filters)
    let $filters                := if ($filtersActive) then $filters else ()
    
    let $compiledScenarios      := if ($decorproject/scenarios) then comp:compileScenarios($decorproject/scenarios, $filters, $runtimeonly) else (<scenarios/>)
    
    let $compiledDatasets       := 
        if ($runtimeonly) then () else comp:compileDatasets($decorproject/datasets, $compiledScenarios, $language, $filters)
    
    (: these get in between in getFullDataset() :)
    let $compiledIds1           := $compiledDatasets/ids
    let $compiledIds2           :=
        <ids>
        {
            for $baseId in $compiledDatasets//terminologyAssociation[@codeSystem][@codeSystemName]
            let $ref    := $baseId/@codeSystem
            group by $ref
            order by $ref
            return if ($compiledIds1/baseId[@id = $ref]) then () else $baseId[1]
        }
        </ids>
    
    let $compiledDatasets       := if (empty($compiledDatasets)) then () else <datasets>{$compiledDatasets/@*, $compiledDatasets/(node() except ids)}</datasets>
        
    let $compiledRules          := comp:compileRules($decorproject/rules, $filters, $runtimeonly)
    
    let $associationsInScope    :=  if ($runtimeonly) then () else ($compiledDatasets//terminologyAssociation | $compiledScenarios//terminologyAssociation)
    let $valueSetAssociations   := $associationsInScope[@valueSet] | $compiledRules//vocabulary[@valueSet] | $compiledRules//answerValueSet[@ref]
    
    (:for valuesets we ALWAYS calculate the right set. With the introduction of template[@ref] it is 
    unpredicatable whether or not all valueSet[@ref] are accounted for. If we leave it up to the users 
    they might be missing one or more valueSet[@ref]:)
    (:  when filtering is off, we 
            - include all valueSet[@id|@ref] regardless of usage
            - include valueSets that are actually in use through *used* terminologyAssociations/templates
        when filtering is on, we 
            - include valueSets that are actually in use through *used* terminologyAssociations/templates
    :)
    let $valueSetsInScope       := 
    (
        if ($filtersActive) then ($filters/valueSet) else (
            for $ref in $decorproject/terminology/valueSet
            let $vsid       := $ref/@id | $ref/@ref
            let $vsed       := $ref/@effectiveDate | $ref/@flexibility
            return
                if ($ref/@id | $valueSetAssociations[@valueSet = $ref] | $valueSetAssociations[@ref = $ref]) then 
                    <valueSet ref="{$vsid}" flexibility="{if (empty($vsed)) then 'dynamic' else $vsed}" statusCode="{($ref/@statusCode)[1]}" name="{($ref/@displayName, $ref/@name)[1]}"/>
                else ()
        )
        ,
        (:  - include valueSets that are actually in use through *used* terminologyAssociations/templates/questionnaires
            
            please note that this step might duplicate valueSet elements created above. 
            this is mitigated in compileTerminology
        :)
        for $ref in $valueSetAssociations
        let $vsid       := $ref/@valueSet | $ref/@ref
        let $vsed       := $ref/@flexibility[not( .= 'dynamic')]
        return
            <valueSet ref="{$vsid}" flexibility="{if (empty($vsed)) then 'dynamic' else $vsed}">
            {
                ($ref/ancestor::*/@ident)[1],
                ($ref/ancestor::*/@url)[1]
            }
            </valueSet>
    )
    let $valueSetsInScope       := 
        <filters filteringActive="{$filtersActive}">
        {
            $filters/@filter,
            (:when filtering is off and we're not doing runtime only, we 
                - always include all valueSet[@id|@ref] regardless of usage
            :)
            for $ref in $valueSetsInScope
            let $vsid       := $ref/@ref
            let $vsed       := $ref/@flexibility
            let $vsident    := $ref/@ident
            let $vsurl      := $ref/@url
            group by $vsid, $vsed, $vsurl, $vsident
            order by $vsid, $vsed, $vsurl, $vsident
            return
                <valueSet ref="{$vsid}" flexibility="{$vsed}">{($ref/@statusCode)[1], ($ref/@name)[1], $vsident, $vsurl}</valueSet>
        }
        </filters>
    
    return 
    if ($testfilters) then (
        <filters>
        {
            $filters/@*,
            $filters/(node() except ignore),
            if ($filtersActive) then (
                for $template in $compiledRules/template
                let $tmid       := $template/@id | $template/@ref
                let $tmed       := $template/@effectiveDate | $template/@flexibility[not(. = 'dynamic')]
                return 
                    <template ref="{$tmid}" flexibility="{($tmed, 'dynamic')[1]}" statusCode="{$template/@statusCode}" name="{($template/@displayName, $template/@name)[1]}"/>
                ,
                $valueSetsInScope/valueSet
            ) else ()
            ,
            $filters/ignore
        }
        </filters>
    ) else (
        <decor>
        {
            for $att in $decorproject/@*
            return
                if ($att/name()=('versionDate','versionLabel','compilationDate','language','deeplinkprefix','deeplinkprefixservices')) 
                then ( (:skip attributes that we will write here:) )
                else if (matches($att/local-name(),'^dummy-[\d]*'))
                then ( (:skip attributes that we will write here:) )
                else ($att)
            ,
            (: hack alert. This forces the serializer to write our 'foreign' namespace declarations. Reported on the exist list :)
            for $ns-prefix at $i in in-scope-prefixes($decorproject)[not(.=('xml'))]
            let $ns-uri := namespace-uri-for-prefix($ns-prefix, $decorproject)
            return
                attribute {QName($ns-uri,concat($ns-prefix,':dummy-',$i))} {$ns-uri}
            ,
            attribute versionDate {substring(string($now), 1, 19)},
            ($decorproject/project/release[@date = $now]/@versionLabel, $decorproject/project/version[@date = $now]/@versionLabel)[not(. = '')][1],
            attribute compilationDate {substring(string($now), 1, 19)},
            attribute language {$language},
            if (string-length($comp:strArtURL) gt 0) then 
                attribute deeplinkprefix {$comp:strArtURL}
            else (),
            if (string-length($comp:strArt3URL) gt 0) then 
                attribute deeplinkprefix3 {$comp:strArt3URL}
            else (),
            if (string-length($comp:strDecorServicesURL) gt 0) then 
                attribute deeplinkprefixservices {$comp:strDecorServicesURL}
            else (),
            if (string-length($comp:strFhirServicesURL) gt 0) then 
                attribute deeplinkprefixservicesfhir {$comp:strFhirServicesURL}
            else ()
            ,
            '&#10;',
            comment {
                '&#10;',
                'This is a compiled version of a DECOR based project', if ($runtimeonly) then 'suitable for the creation of a runtime environment' else (), '. Compilation date: ', $now ,'&#10;',
                'PLEASE NOTE THAT ITS ONLY PURPOSE IS TO FACILITATE HTML AND SCHEMATRON GENERATION. HENCE THIS IS A ONE OFF FILE UNSUITED FOR ANY OTHER PURPOSE','&#10;',
                if ($runtimeonly) then () else 'Compilation process calls getFullDataSetTree where all inheritance of concepts from repositories is resolved&#10;',
                'Compilation process leaves valueSet[@ref] and template[@ref] as-is but adds, if available, the valueSet/template (versions) it references. These are marked with valueSet[@referencedFrom, @ident and/or @url]','&#10;',
                if ($runtimeonly) then (
                    'Compilation process skips datasets', '&#10;',
                    'Compilation process skips issues', '&#10;',
                    'Compilation process skips terminology/terminologyAssociation', '&#10;',
                    'Compilation process skips dataset references and concepts in transaction/representingTemplate, copying only references to templates', '&#10;',
                    'Compilation process copies ids/baseId and ids/defaultBaseId.'
                ) else (
                    'Compilation process tries to find names for any OIDs referenced in the project but not yet in ids/id, and adds an entry if a name is found.', '&#10;'
                )
            }
        }
        {
            let $compiledTerminology    := 
                if ($decorproject[terminology]) then 
                    comp:compileTerminology($decorproject/terminology, $valueSetsInScope, $runtimeonly)
                else (
                    comp:compileTerminology(<terminology/>, $valueSetsInScope, $runtimeonly)
                )
            
            return
            for $node in $decorproject/node()
            return
                if ($node/name()='project') then (
                    comp:compileProject($node, $language)
                ) else 
                if ($node/name()='datasets') then (
                    if ($runtimeonly) then (<datasets/>) else $compiledDatasets
                ) else 
                if ($node/name()='scenarios') then (
                    $compiledScenarios
                ) else 
                if ($node/name()='terminology') then (
                    $compiledTerminology
                ) else 
                if ($node/name()='ids') then (
                    comp:compileIds($node, $compiledIds1 | $compiledIds2, ($compiledDatasets | $compiledTerminology | $compiledRules), $runtimeonly)
                ) else 
                if ($node/name()='rules') then (
                    (: In order for downstream tooling to produce meaningful links you need more info on 
                    the relationship element as the referred template may very well not be in the set :)
                    let $templateMap    :=
                        map:merge(
                            for $t in $compiledRules//relationship[@template]
                            let $tmid   := $t/@template
                            let $tmed   := $t/@flexibility[. castable as xs:dateTime]
                            let $tmided := concat($tmid, $tmed)
                            group by $tmided
                            return (
                                let $template   := $get:colDecorData//template[@id = $tmid[1]] | $get:colDecorCache//template[@id = $tmid[1]]
                                let $tmed       := if ($tmed[1] castable as xs:dateTime) then $tmed[1] else string(max($template/xs:dateTime(@effectiveDate)))
                                let $template   := ($template[@effectiveDate = $tmed])[1]
                                return
                                if ($template) then 
                                    map:entry($tmided, 
                                        <template>
                                        {
                                            $template/@*, 
                                            if ($template/@url) then () else 
                                            if ($template/ancestor::cacheme) then attribute url {$template/ancestor::cacheme/@bbrurl} else (
                                                attribute url {adserver:getServerURLServices()}
                                            ),
                                            if ($template/@ident) then () else 
                                            if ($template/ancestor::cacheme) then attribute ident {$template/ancestor::cacheme/@bbrident} else (
                                                attribute ident {$template/ancestor::decor/project/@prefix}
                                            )
                                        }
                                        </template>
                                    )
                                else ()
                            )
                        )
                    let $questionnaireMap :=
                        map:merge(
                            for $t in $compiledRules//relationship[@type = 'ANSW'] | $compiledRules//relationship[@type = 'DRIV']
                            let $tmid   := $t/@ref
                            let $tmed   := $t/@flexibility[. castable as xs:dateTime]
                            let $tmided := concat($tmid, $tmed)
                            group by $tmided
                            return (
                                let $object       := 
                                    if ($t/@type = 'DRIV') then
                                        $get:colDecorData//transaction[@id = $tmid[1]][@effectiveDate = $tmed[1]] | $get:colDecorCache//transaction[@id = $tmid[1]][@effectiveDate = $tmed[1]]
                                    else (
                                        $get:colDecorData//questionnaire[@id = $tmid[1]][@effectiveDate = $tmed[1]] | $get:colDecorCache//questionnaire[@id = $tmid[1]][@effectiveDate = $tmed[1]]
                                    )
                                return
                                if ($object) then 
                                    map:entry($tmided, 
                                        element {name($object[1])}
                                        {
                                            $object/@*, 
                                            if ($object/@url) then () else 
                                            if ($object/ancestor::cacheme) then attribute url {$object/ancestor::cacheme/@bbrurl} else (
                                                attribute url {adserver:getServerURLServices()}
                                            ),
                                            if ($object/@ident) then () else 
                                            if ($object/ancestor::cacheme) then attribute ident {$object/ancestor::cacheme/@bbrident} else (
                                                attribute ident {$object/ancestor::decor/project/@prefix}
                                            )
                                        }
                                    )
                                else ()
                            )
                        )
                    
                    return
                        element {name($compiledRules)} {
                            for $node in $compiledRules/node()
                            return
                            if ($node[name() = 'template'][relationship/@template]) then
                                element {name($node)} {
                                    $node/@*,
                                    for $tnode in $node/node()
                                    return
                                        if ($tnode[name() = 'relationship'][@template]) then
                                            element relationship {
                                                $tnode/(@* except (@tmid|@tmname|@tmdisplayName|@tmeffectiveDate|@tmexpirationDate|@tmstatusCode|@tmversionLabel|@linkedartefactmissing|@url|@ident)),
                                                let $tm := map:get($templateMap, concat($tnode/@template, $tnode/@flexibility[. castable as xs:dateTime]))
                                                return
                                                if (empty($tm)) then (
                                                    attribute {'linkedartefactmissing'} {empty($tm)}
                                                ) else (
                                                    attribute {'tmid'} {$tm/@id},
                                                    attribute {'tmname'} {$tm/@name},
                                                    attribute {'tmdisplayName'} {if ($tm[@displayName]) then $tm/@displayName else $tm/@name},
                                                    if ($tm/@effectiveDate) then attribute {'tmeffectiveDate'} {$tm/@effectiveDate} else (),
                                                    if ($tm/@expirationDate) then attribute {'tmexpirationDate'} {$tm/@expirationDate} else (),
                                                    if ($tm/@statusCode) then attribute {'tmstatusCode'} {$tm/@statusCode} else (),
                                                    if ($tm/@versionLabel) then attribute {'tmversionLabel'} {$tm/@versionLabel} else (),
                                                    $tm/@url,
                                                    $tm/@ident
                                                ),
                                                $tnode/*
                                            }
                                        else (
                                            $tnode
                                        )
                                }
                            else
                            if ($node[name() = 'questionnaire'][relationship]) then
                                element {name($node)} {
                                    $node/@*,
                                    for $tnode in $node/node()
                                    return
                                        if ($tnode[name() = 'relationship']) then
                                            element relationship {
                                                $tnode/@type, $tnode/@ref, $tnode/@flexibility,
                                                let $tm := map:get($questionnaireMap, concat($tnode/@ref, $tnode/@flexibility[. castable as xs:dateTime]))
                                                let $pf := name($tm)
                                                return
                                                if (empty($tm)) then (
                                                    attribute {'linkedartefactmissing'} {empty($tm)}
                                                ) else (
                                                    for $att in $tm/(@id, @effectiveDate, @expirationDate, @statusCode, @versionLabel)
                                                    return
                                                        attribute {$pf || upper-case(substring(name($att), 1, 1)) || substring(name($att), 2)} {$att}
                                                    ,
                                                    $tm/@url,
                                                    $tm/@ident,
                                                    $tm/name
                                                ),
                                                $tnode/*
                                            }
                                        else (
                                            $tnode
                                        )
                                }
                            else (
                                $node
                            )
                        }
                ) else 
                if ($node/name()='issues') then (
                    if ($runtimeonly) then (<issues/>) else comp:compileIssues($node, $language, ())
                ) else (
                    $node
                )
        }
        </decor>
    )
};

(:~ Returns a matching compilation or create one (unless $forceCompile is empty, in which case it only returns a compilation if found and empty if there is not)

    @param $decor           - The DECOR project to compile
    @param $now             - The timestamp to add to the compilation, if empty uses current-dateTime()
    @param $language        - The project language to compile for or '*' for all, if empty uses project default language
    @param $filters         - The filters to apply if any. Note: pre-process any filters through comp:getFinalCompilationFilters($decor, $filters) before sending them here
    @param $testfilters     - Boolean value that determines if this function returns the fully processed filters or the DECOR compilation itself. Fully processed filters also contain terminology
    @param $runtimeonly     - Boolean value that guides the compilation for runtime usage or not. Runtime compilations don't have any datasets and carry only transactions that point to a template
    @param $forceCompile    - Boolean value that if true prohibits checking for an existing compilation to return
    @return Compiled DECOR project or fully compiled filters or nothing
:)
declare function comp:getCompiledResult($decor as element(decor), $now as xs:dateTime?, $language as xs:string?, $filters as element(filters)?, $testfilters as xs:boolean, $runtimeonly as xs:boolean, $forceRecompile as xs:boolean?) as element()? {
    let $filtersActive  := comp:isCompilationFilterActive($filters)
    let $filters        := if ($filtersActive) then $filters else ()
    let $colTmp         := collection($get:strDecorTemp)
    let $decorPrefix    := $decor/project/@prefix
    let $decorId        := $decor/project/@id
    let $now            := if (empty($now)) then current-dateTime() else $now
    let $lastmodified   := xmldb:last-modified(util:collection-name($decor), util:document-name($decor))
    let $language       := if ($decor/project/name[@language = $language] or $language = '*') then $language else $decor/project/@defaultLanguage
    
    (: if forceCompile it does not matter if there is a previous result. we are forced to make a new one regardless :)
    let $compile        := 
        if ($forceRecompile) then () else (
            comp:getTempCompilations($decorId)[@language = $language][@testfilters = $testfilters][@runtimeonly = $runtimeonly][xs:dateTime(@compileDate) ge $lastmodified]
        )
    let $compile        :=
        (: if there is no compile, don't bother further selection :)
        if (empty($compile)) then () else
        if ($filtersActive) then (
            (: if we are asked to filter, then look for a compilation that was based on the same filters. 
                Note that deep-equal(filters, $filters) does not work because of exist-db white space handling so we write it to exist-db first before compare :)
            let $filterstring   := string-join($filters/*[@ref]/concat(@ref, @flexibility), ' ')
            return
                $compile[string-join(filters/*[@ref]/concat(@ref, @flexibility), ' ') = $filterstring]
        )
        else
            (: if we are not asked to filter then return unfiltered compilations only :)
            $compile[not(comp:isCompilationFilterActive(filters))]
    
    return
        (: return the first of the compilations that carries the latest date :)
        if ($compile) then $compile[@compileDate = max($compile/xs:dateTime(@compileDate))][1] else
        if (empty($forceRecompile)) then ( 
            (: don't return anything to let caller know we did not find what he was looking for. 
                Useful when caller wants to check for full compile in case there is one, but really needs runtime or something else only :) 
        ) else (
            let $ftemp  := concat($decorPrefix, replace(substring(string(current-dateTime()), 1, 19), '[:\-]', ''), '.xml')
            return
                try {
                    (: write initial file so subsequent calls know we're busy :)
                    let $compilefile            := xmldb:store($get:strDecorTemp, $ftemp, 
                        (:<compile id="{$decorId}" language="{$language}" testfilters="{$testfilters}" runtimeonly="{$runtimeonly}" compileDate="{$now}" compilationFinished=""/>:)
                        <compile id="{$decorId}" language="{$language}" testfilters="{$testfilters}" runtimeonly="{$runtimeonly}" compileDate="{$now}"/>
                    )
                    let $doc                    := doc($compilefile)/compile
                    let $compilationresult      := comp:compileDecor($decor, $language, $now, $filters, $testfilters, $runtimeonly)
                    let $store                  := update insert $compilationresult into $doc
                    let $store                  := if (comp:isCompilationFilterActive($filters)) then update insert $filters into $doc else ()
                    let $store                  := update insert attribute compilationFinished {format-dateTime(current-dateTime(), '[Y0001]-[M01]-[D01]T[H01]:[m01]:[s01]')} into $doc
                    
                    return doc($compilefile)/compile
                }
                catch * {
                    let $remove             := try { xmldb:remove($get:strDecorTemp, $ftemp) } catch * {()}
                    
                    return
                        error(xs:QName('comp:CompilationError'), concat($err:code, ': ', $err:description, ' in module: ', $err:module, ' (', $err:line-number, ' ', $err:column-number, ')'))
                }
        )
    
};

declare function comp:getTempCompilations($projectId as xs:string*) as element(compile)* {
    collection($get:strDecorTemp)/compile[@id = $projectId]
};

declare function comp:getCompilationFilterFile($decor as element(decor)) as element(filters)? {
    collection(util:collection-name($decor))/filters[util:document-name(.) = 'filters.xml']
};

(:~ Retrieves data from filters.xml file adjacent to wherever the DECOR project file is. Reads @filter from rootelement. This may be "off" or "on".
    
    New:
    <filters filter="on|off" filterId="filter/@id">
        <filters id="1" label="Filter set name" created="date" modified="date" [active=""]>
            <transaction ref="..." flexibility="..."/>
            ...
        </filters>
    </filters>
    
    Old school:
    <filters filter="on|off" [label="..."]>
        <transaction ref="..."/>
        ...
    </filters>
    
    @return contents of filters.xml file
:)
declare function comp:getCompilationFilterSet($decor as element(decor)) as element(filters)? {
    comp:prepareFilterSet(comp:getCompilationFilterFile($decor))
};

(:~ Filters are active if filter attribute is not 'off', and if there are things to filter on, and there is no filterId set or if it is, it matches some filters id :)
declare function comp:isCompilationFilterActive($filters as element(filters)?) as xs:boolean {
    empty($filters[@filter = 'off']) and $filters[*] and (empty($filters/@filterId) or $filters/@filterId = $filters/filters/@id)
};

(:~ Retrieves data from filters.xml file adjacent to wherever the DECOR project file is. Reads @filter from rootelement. This may be "off" or "on".
    @return If @filter="off" then <filters filter="off"/> otherwise <filters filter="on">...</filters> where "..." is the calculated list of artifacts to process starting from <transaction .../> elements
:)
declare function comp:getCompilationFilters($decor as element(decor), $filterset as element(filters)?, $filterid as xs:string?) as element(filters)? {
let $filterset      := if ($filterset) then $filterset else comp:getCompilationFilterSet($decor)
let $filterActive   := comp:isCompilationFilterActive($filterset)
let $filterid       := if (empty($filterid)) then $filterset/@filterId else $filterid

let $filters        := 
    if ($filterActive) then 
        if (empty($filterset/filters)) then
            (: old school. filters.xml directly contains the filters. we did not support filterset originally. 
                how did we get the filterId then? RetrieveProject for example will offer this id if none is available. :)
            $filterset
        else
            $filterset/filters[@id = $filterid]
    else ()

let $filters        := 
    <filters>
    {
        $filters/(@* except @filter),
        attribute filter {if ($filterActive and $filters) then 'on' else 'off'},
        $filters/node()
    }
    </filters>

return
    $filters

};

declare function comp:updateCompilationFilterSet($decor as element(decor), $filterset as element(filters)) as element(filters) {
let $updatedFilters     := comp:prepareFilterSet($filterset)

let $store              := xmldb:store(util:collection-name($decor), 'filters.xml', $updatedFilters)
let $tt                 := sm:chmod(xs:anyURI($store), 'rw-rw-r--')

return
    comp:getCompilationFilterSet($decor)
};

declare function comp:updateCompilationFilters($decor as element(decor), $filters as element(filters)) as element(filters) {
let $storedFilterSet    := comp:getCompilationFilterFile($decor)
let $updatedFilters     := comp:prepareFilters($storedFilterSet, $filters)
let $storedFilters      := $storedFilterSet/filters[@id = $updatedFilters/@id]

let $store              := 
    if ($storedFilters) then
        update replace $storedFilters with $updatedFilters
    else
    if ($storedFilterSet) then
        update insert $updatedFilters into $storedFilterSet
    else (
        let $ttt        := xmldb:store(util:collection-name($decor), 'filters.xml', comp:prepareFilterSet(<filters filter="on" filterId="{$updatedFilters/@id}">{$updatedFilters}</filters>))
        let $tt         := sm:chmod(xs:anyURI($ttt), 'rw-rw-r--')
        
        return $ttt
    )
let $delete-old-style   :=
    update delete comp:getCompilationFilterFile($decor)/(* except (filters | ignore))

return
    if ($filters[@filter = 'on'] | $filters[@filter = 'off']) then 
        comp:updateCompilationFiltersActivation($decor, $filters/@filter = 'on') 
    else (
        comp:getCompilationFilterSet($decor)
    )
};

declare function comp:updateCompilationFiltersActivation($decor as element(decor), $activate as xs:boolean) as element(filters) {
let $storedFilterSet    := comp:getCompilationFilterFile($decor)
let $isactive           := if ($activate) then 'on' else 'off'

let $store          := 
    if ($storedFilterSet[@filter]) then
        update value $storedFilterSet/@filter with $isactive
    else
    if ($storedFilterSet) then
        update insert attribute filter {$isactive} into $storedFilterSet
    else (
        (: cannot activate unless you have content... should we error? :)
        let $ttt        := xmldb:store(util:collection-name($decor), 'filters.xml', <filters filter="{$isactive}"/>)
        let $tt         := sm:chmod(xs:anyURI($ttt), 'rw-rw-r--')
        
        return $ttt
    )

return
    comp:getCompilationFilterSet($decor)
};

(:~ Return <filters filter="on|off" (filterId="n")>(<filters>...</filters>)</filters> :)
declare function comp:prepareFilterSet($filterset as element(filters)?) as element(filters) {
    let $filtersActive      := comp:isCompilationFilterActive($filterset)
    let $now                := substring(string(current-dateTime()), 1, 19)
    let $newest             := max($filterset/filters[@modified castable as xs:dateTime]/xs:dateTime(@modified))
    
    return
        <filters>
        {
            attribute filter {if ($filtersActive) then 'on' else 'off'}
            ,
            if ($filterset[@filterId = filters/@id]) then 
                $filterset/@filterId
            else
            if ($filterset[filters/@id]) then 
                attribute filterId {($filterset/filters[@modified = $newest]/@id, $filterset/filters/@id)[1]}
            else
            if ($filterset[*]) then
                attribute filterId {1}
            else (
                attribute filterId {''}
            )
        }
        {
            let $prepareFromOldSchool   :=
                if ($filterset[* except filters]) then (
                    let $newid      := if (empty($filterset/filters[@id])) then 0 else max($filterset/filters/xs:integer(@id))
                    let $newid      := $newid + 1
                    
                    return
                        comp:prepareFilters($filterset, 
                            <filters id="{$newid}" label="{($filterset/@label[not(. = '')], concat('Label ', $newid))[1]}" created="{$now}" modified="{$now}">
                            {
                                $filterset/(* except (filters | ignore))
                            }
                            </filters>
                        )
                )
                else ()
            
            for $f in ($prepareFromOldSchool, $filterset/filters)
            return
                comp:prepareFilters($filterset, $f)
        }
        </filters>
};

declare function comp:prepareFilters($filterset as element(filters)?, $filters as element(filters)) as element(filters) {
    let $now        := substring(string(current-dateTime()), 1, 19)
    let $newid      := if (empty($filters/filters[@id] | $filters/filters[@id = '1'])) then 0 else (
                            max($filters/filters/xs:integer(@id))
                       )
    let $newid      := $newid + 1
    
    return
        <filters>
        {
            if ($filters/@id)       then $filters/@id       else attribute id {$newid},
            if ($filters/@label)    then $filters/@label    else attribute label {concat('Label', ($filters/@id, $newid)[1])},
            if ($filters/@created)  then $filters/@created  else attribute created {$now},
            if ($filters/@modified) then $filters/@modified else attribute modified {$now},
            for $node in $filters/(* except (filters | ignore))
            return
                if ($node[@ref][@flexibility castable as xs:dateTime]) then $node else
                if ($node[@ref]) then (
                    let $object     := 
                        if ($node[self::transaction]) then art:getTransaction($node/@ref, 'dynamic')/ancestor-or-self::transaction else
                        if ($node[self::scenario]) then art:getScenario($node/@ref, 'dynamic') else
                        if ($node[self::dataset]) then art:getDataset($node/@ref, 'dynamic') else
                        if ($node[self::template]) then templ:getTemplateById($node/@ref, 'dynamic')/template/template[@id]
                        else ()
                    return
                        if ($object) then
                            element {name($node)} {
                                $node/@ref,
                                attribute flexibility {$object[1]/@effectiveDate},
                                attribute statusCode {$object[1]/@statusCode},
                                attribute name {($object/name, $object/@displayName, $object/@name)[1]}
                            }
                        else ($node)
                )
                else ($node)
        }
        </filters>
};

(:~ Process and return contents of filters.xml. Normally this file contains transaction references leading to scenarios, datasets and templates. These in turn lead to valuesets.
    Use cases to support:
    1. Do datasets based on id (includes valuesets and terminologyAssociations, excludes any scenarios)
    2. Do full scenarios based on id (includes linked datasets, valuesets, terminologyAssociations, templates)
    3. Do transaction (groups) based on id (includes linked datasets, valuesets, terminologyAssociations, templates)
    4. ...
    
   Filtering actually works on the inclusion mechanism. If it is not in the filters file, it should not be compiled.
:)
declare function comp:getFinalCompilationFilters($decor as element(decor), $filters as element(filters)?) as element(filters)? {

let $filterSetting  := if (comp:isCompilationFilterActive($filters)) then ('on') else ('off')

(: get every transaction that is being called out :)
let $transactions           := 
    for $ref in $filters/transaction
    let $id     := $ref/@ref
    let $ed     := $ref/@flexibility[not(. = 'dynamic')]
    group by $id, $ed
    return 
        art:getTransaction($id, $ed)/ancestor-or-self::transaction

(: get every scenario that is being called out :)
let $scenarios              := (
    $transactions/ancestor::scenario
    ,
    for $ref in $filters/scenario
    let $id     := $ref/@ref
    let $ed     := $ref/@flexibility
    group by $id, $ed
    return (
        let $sc := art:getScenario($id, $ed)
        return 
            if ($transactions/ancestor::scenario[@id = $sc/@id][@effectiveDate = $sc/@effectiveDate]) then () else $sc
    )
)
(: get every transaction that is being called out. If nothing is called out, apparently we want every transaction under the scenarios :)
let $transactions           := if ($transactions) then $transactions else $scenarios//transaction

(: get every dataset that is being called out or attached to a transaction :)
let $datasets               := 
    for $ref in $filters/dataset | $transactions//representingTemplate[@sourceDataset]
    let $id     := $ref[self::dataset]/@ref | $ref/@sourceDataset
    let $ed     := $ref[self::dataset]/@flexibility[not(. = 'dynamic')] | $ref/@sourceDatasetFlexibility[not(. = 'dynamic')]
    group by $id, $ed
    return 
        art:getDataset($id, $ed)

(: filter templates :)
let $templates              :=
    for $ref in $transactions//representingTemplate[@ref]
    let $id     := $ref/@ref
    let $ed     := $ref/@flexibility[not(. = 'dynamic')]
    group by $id, $ed
    return (
        templ:getTemplateByRef($id, ($ed, 'dynamic')[1], $decor/project/@prefix)/template/template[@id]
    )

(: filter questionnaires :)
let $questionnaires         :=
    for $ref in $transactions//representingTemplate[@representingQuestionnaire]
    let $id     := $ref/@representingQuestionnaire
    let $ed     := $ref/@representingQuestionnaireFlexibility[not(. = 'dynamic')]
    group by $id, $ed
    return (
        if ($ed castable as xs:dateTime) then 
            $get:colDecorData//questionnaire[@id = $id][@effectiveDate = $ed]
        else (
            let $q  := $get:colDecorData//questionnaire[@id = $id]
            return
                $q[@effectiveDate = string(max($q/xs:dateTime(@effectiveDate)))]
        )
    )
    
(: rewrite the filters so we now know what it is exactly that we need to do :)
let $finalFilters :=
    (
        for $ref in $datasets | $scenarios[self::dataset]
        let $id     := $ref/@id
        let $ed     := $ref/@effectiveDate
        group by $id, $ed
        order by $id, $ed
        return
            <dataset ref="{$id}" flexibility="{$ed}" statusCode="{$ref[1]/@statusCode}" name="{$ref[1]/name[1]}"/>
        ,
        for $ref in $scenarios[self::scenario] | $transactions/ancestor-or-self::scenario
        let $id     := $ref/@id
        let $ed     := $ref/@effectiveDate
        group by $id, $ed
        order by $id, $ed
        return
            <scenario ref="{$id}" flexibility="{$ed}" statusCode="{$ref[1]/@statusCode}" name="{$ref[1]/name[1]}"/>
        ,
        for $ref in $transactions/descendant-or-self::transaction
        let $id     := $ref/@id
        let $ed     := $ref/@effectiveDate
        group by $id, $ed
        order by $id, $ed
        return
            <transaction ref="{$id}" flexibility="{$ed}" statusCode="{$ref[1]/@statusCode}" name="{$ref[1]/name[1]}"/>
        ,
        for $ref in $templates
        let $id     := $ref/@id | $ref/@ref
        let $ed     := $ref/@effectiveDate
        group by $id, $ed
        order by $id, $ed
        return (
            <template ref="{$id}" flexibility="{$ed}" statusCode="{$ref[1]/@statusCode}" name="{($ref/@displayName, $ref/@name)[1]}"/>
        )
        ,
        for $ref in $questionnaires
        let $id     := $ref/@id | $ref/@ref
        let $ed     := $ref/@effectiveDate
        group by $id, $ed
        order by $id, $ed
        return (
            <questionnaire ref="{$id}" flexibility="{$ed}" statusCode="{$ref[1]/@statusCode}" name="{($ref/@displayName, $ref/@name, $ref/name)[1]}"/>
        )
    )
return
    <filters filter="{$filterSetting}">
    {
        if ($filterSetting='on') then (
            $filters/@label
            ,
            $finalFilters
            ,
            '&#10;'
            ,
            if ($finalFilters[@ref]) then (
                <ignore>{
                    comment {'Any reference below here will be ignored based on applicable filters:'}
                    ,
                    for $node in $decor//dataset | $decor//scenario | $decor//transaction
                    let $type   := name($node)
                    let $id     := $node/@id
                    let $ed     := $node/@effectiveDate
                    order by $type, $id, $ed
                    return
                        if ($finalFilters[@ref = $node/@id][@flexibility = $node/@effectiveDate]) then () else (
                            <filter type="{$node/name()}" dref="{$node/@id}" dflexibility="{$node/@effectiveDate}" dstatusCode="{$node/@statusCode}" dname="{($node/name[1], $node/@displayName, $node/@name)[1]}"/>
                        )
                }</ignore>
            )
            else if ($filters) then (
                <ignore>{
                    comment {'WARNING: Filters where defined, but the project doesn''t match any of the referred artifacts.'}
                    ,
                    $filters/node()
                }</ignore>
            )
            else ()
        )
        else (
            comment {'Filtering is off, everything will be included'}
        )
    }
    </filters>
};