Metadata Lineage from Eagle MDC to Ontology - field attributes and element definitions

Example of the RTR to generate dataset and mashup file via eagle_ml-2-0_extract_service_control_message stream

<EagleML xmlns="http://www.eagleinvsys.com/2011/EagleML-2-0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="RunTaskRequest" eaglemlVersion="2-0" xsi:schemaLocation="http://www.eagleinvsys.com/2011/EagleML-2-0 eagleml-main-2-0.xsd" eaglemlType="RunTaskRequest">
<header>
    <messageId>B70DHO72GCLFRHGI</messageId>
    <sentBy>http://www.eagleinvsys.com</sentBy>
    <sendTo>http://www.eagleinvsys.com/eagle_default_ruleservice</sendTo>
</header>
	<taskIdentifier>
		<correlationId>model_all</correlationId>
		<businessTaskId>model_all</businessTaskId>
	</taskIdentifier>
	<taskTypeEnum>NEW</taskTypeEnum>
	<synchronousExecution>yes</synchronousExecution>
	<taskParameters>
		<taskParameter>
			<name>ServiceName</name>
			<value>Metadata</value>
		</taskParameter>
		<taskParameter>
			<name>ServiceMethodName</name>
			<dataType>S</dataType>
			<value>ontologymetadata</value> 
		</taskParameter>
		<taskParameter>
			<name>ResourceName</name>
			<value>ALL</value>
		</taskParameter>
	</taskParameters>
</EagleML>

Generated dataset (tpe\dynamic\metadata\core\datasets\ontology_metadata.json):

{
    "_id": "ontology_metadata",
    "dataframes": [
        {
            "_id": "ontology_metadata",
            "type": "dftFile",
            "group_name": "Ontology",
            "description": "Ontology metadata",
            "maxrows": 9999999,
            "source_sink": {
                "sink_type": "ssFileType",
                "sink_params": {
                    "file_name": "estar/tpe/dynamic/metadata/core/ontology/templates/model_ALL.csv",
                    "mode": "r"
                },
                "source_format_dialect": {
                    "lineterminator": "\n",
                    "delimiter": "|~",
                    "headerlines": 1,
                    "headerat": 1,
                    "dialectname": "simplesplit"
                }
            },
            "format": "csv",
            "vocabulary": {
                "FIELD_PATH": "FIELD_PATH",
                "FIELD_DEFINITION": "FIELD_DEFINITION"
            },
            "unique_key": [
                {
                    "field": "FIELD_PATH",
                    "expression": "|FIELD_PATH|"
                }
            ],
            "source_query_policy": {
                "cache_mode": "none",
                "method": "file",
                "base_template": "",
                "full": {
                    "filter_params": {
                        "filter_definitions": {}
                    }
                },
                "filter_params": {
                    "filter_definitions": {}
                }
            },
            "cache_policy": {
                "type": "aside",
                "cache_provider": "cache-manager-file",
                "storage_method": "keyvalue",
                "compression_codec": "N",
                "encryption_codec": "N",
                "format_dialect": "mashup_csv_dialect",
                "ttl_delta": 0,
                "ttl_unique": 60
            }
        }
    ]
}

The mashup file (model_ALL.csv) contains two columns FIELD_PATH and FIELD_DEFINITION. This file is located: estar/tpe/dynamic/metadata/core/ontology/templates/model_ALL.csv
These definitions are ontology definitions from the folder(tpe\dynamic\metadata\core\ontology\definition) for canonical elements from the corresponding files:

The Field attribute extract uses the following query to pull data from Eagle environment:

SELECT RTRIM(MASTERTABLE.CALC_TYPE) CALC_TYPE,
RTRIM(MASTERTABLE.COMMENTS) COMMENTS,
RTRIM(MASTERTABLE.DATABASE_NAME) DATABASE_NAME,
TO_CHAR(MASTERTABLE.DENOMINTOR_ID) DENOMINTOR_ID,
RTRIM(MASTERTABLE.FIELD_ATTRIBUTE_DESCRIPTION) FIELD_ATTRIBUTE_DESCRIPTION,
TO_CHAR(MASTERTABLE.FIELD_ATTRIBUTE_ID) FIELD_ATTRIBUTE_ID,
TO_CHAR(MASTERTABLE.FIELD_ATTRIBUTE_PROCESS_ID) FIELD_ATTRIBUTE_PROCESS_ID,
RTRIM(MASTERTABLE.FIELD_FORMAT) FIELD_FORMAT,
RTRIM(MASTERTABLE.FIELD_INDICATOR) FIELD_INDICATOR,
RTRIM(MASTERTABLE.FIELD_NAME) FIELD_NAME,
RTRIM(MASTERTABLE.FIELD_TYPE) FIELD_TYPE,
TO_CHAR(MASTERTABLE.NUMERATOR_ID) NUMERATOR_ID,
TO_CHAR(MASTERTABLE.NUMERATOR_ID2) NUMERATOR_ID2,
TO_CHAR(MASTERTABLE.REF_CODE_ID) REF_CODE_ID,
TO_CHAR(MASTERTABLE.SRC_INTFC_INST) SRC_INTFC_INST,
RTRIM(MASTERTABLE.STAT_FIELD) STAT_FIELD,
RTRIM(MASTERTABLE.TABLE_NAME) TABLE_NAME,
CASE WHEN MASTERTABLE.UPDATE_DATE IS NOT NULL THEN TO_CHAR(CAST(MASTERTABLE.UPDATE_DATE AS TIMESTAMP WITH TIME ZONE), 'YYYY-MM-DD') || 'T' || TO_CHAR(CAST(MASTERTABLE.UPDATE_DATE AS TIMESTAMP WITH TIME ZONE), 'HH24:MI:SSTZH:TZM') ELSE NULL END UPDATE_DATE,
RTRIM(MASTERTABLE.UPDATE_SOURCE) UPDATE_SOURCE,
FIELD_ATTRIBUTE_DETAIL.ATTRIB_USER_DATA ATTRIB_USER_DATA,
RTRIM(FIELD_ATTRIBUTE_DETAIL.CURRENCY_PROCESS) CURRENCY_PROCESS,
RTRIM(FIELD_ATTRIBUTE_DETAIL.FIELD_CATEGORY) FIELD_CATEGORY,
RTRIM(FIELD_ATTRIBUTE_DETAIL.FIELD_IDENTIFIER) FIELD_IDENTIFIER,
RTRIM(FIELD_ATTRIBUTE_DETAIL.FIELD_VALUE_TYPE) FIELD_VALUE_TYPE,
RTRIM(FIELD_ATTRIBUTE_DETAIL.READ_ONLY) READ_ONLY,
TO_CHAR(FIELD_ATTRIBUTE_DETAIL.SCOPE) SCOPE,
RTRIM(FIELD_ATTRIBUTE_PROCESS.DATABASE_NAME) DATABASE_NAME_1,
TO_CHAR(FIELD_ATTRIBUTE_PROCESS.FIELD_ATTRIBUTE_PROCESS_ID) FIELD_ATTRIBUTE_PROCESS_ID_1,
RTRIM(FIELD_ATTRIBUTE_PROCESS.PARAMETER_LIST) PARAMETER_LIST,
RTRIM(FIELD_ATTRIBUTE_PROCESS.PROCESS_DESCRIPTION) PROCESS_DESCRIPTION,
RTRIM(FIELD_ATTRIBUTE_PROCESS.PROCESS_TYPE) PROCESS_TYPE,
RTRIM(FIELD_ATTRIBUTE_PROCESS.STORED_PROCEDURE_NAME) STORED_PROCEDURE_NAME,
RTRIM(FIELD_PROCESS_DEFINITION.PARAMETER_TYPE) PARAMETER_TYPE,
RTRIM(FIELD_PROCESS_DEFINITION.PARAMETER_VALUE) PARAMETER_VALUE,
TO_CHAR(FIELD_PROCESS_DEFINITION.SEQ_NO) SEQ_NO,
FIELD_PROCESS_DEFINITION.USER_DATA USER_DATA
FROM RULESDBO.FIELD_ATTRIBUTES MASTERTABLE
LEFT JOIN RULESDBO.FIELD_ATTRIBUTE_DETAIL FIELD_ATTRIBUTE_DETAIL ON MASTERTABLE.FIELD_ATTRIBUTE_ID = FIELD_ATTRIBUTE_DETAIL.FIELD_ATTRIBUTE_ID
LEFT JOIN RULESDBO.FIELD_ATTRIBUTE_PROCESS FIELD_ATTRIBUTE_PROCESS ON MASTERTABLE.FIELD_ATTRIBUTE_PROCESS_ID = FIELD_ATTRIBUTE_PROCESS.FIELD_ATTRIBUTE_PROCESS_ID
LEFT JOIN RULESDBO.FIELD_PROCESS_DEFINITION FIELD_PROCESS_DEFINITION ON MASTERTABLE.FIELD_ATTRIBUTE_ID = FIELD_PROCESS_DEFINITION.FIELD_ATTRIBUTE_ID and FIELD_PROCESS_DEFINITION.USER_DATA is not NULL

WHERE (1=1)

Output record sample:

	<referenceTransaction>
		<header>
			<objectType>FieldAttribute</objectType>
		</header>
		<fieldAttribute>
			<calcType>di</calcType>
			<databaseName>TRADES</databaseName>
			<fieldAttributeDescription>i Trade Amount</fieldAttributeDescription>
			<fieldAttributeId>153</fieldAttributeId>
			<fieldIndicator>t</fieldIndicator>
			<fieldName>TRADE_AMOUNT</fieldName>
			<fieldType>num</fieldType>
			<tableName>TRADE</tableName>
			<fieldFormat>2</fieldFormat>
			<fieldIdentifier>I TRADE AMOUNT</fieldIdentifier>
			<scope>2</scope>
			<readOnly>Y</readOnly>
			<fieldAttributeOntologyDescription>The trade amount associated with this disposal lot. The proceeds of the close event.</fieldAttributeOntologyDescription>
		</fieldAttribute>
	</referenceTransaction>

Value in mashup file:

model_ALL.csv
TRADE.TRADE_AMOUNT|~The trade amount associated with this disposal lot. The proceeds of the close event.

Changes in processing rule for fieldattribute.

Now it contains logic to perform mashup, this logic is generated based on ontology. Value for new field is populated from mashup file generated based on ontology elements definition.

"register_for_mashups": [
    {
        "_alias": "ontology_metadata",
        "dataset": "ontology_metadata",
        "dataframe": "ontology_metadata",
        "keyexpr": "|FIELD_PATH|",
        "columns": "*"
    }
],
"mashups": {
    "ontology_metadata": "select first FIELD_DEFINITION from ontology_metadata where (|mainds.TABLE_NAME| + '.' + |mainds.FIELD_NAME|)=(|FIELD_PATH|)"
}