Versions Compared

Old Version 4

changes.mady.by.user Pamela Totas

Saved on

compared with

New Version 5

changes.mady.by.user Pamela Totas

Saved on

Key

  • This line was added.
  • This line was removed.
  • Formatting was changed.

...

Code Block
{
  "resourcename": "legalentityid",
  "dbprovider": "snowflake",
  "schemadriftmode": "ignore"
}

VALIDATE

...

Schema Drift Mode

Schema drift mode VALIDATE is detecting any schema changes for resource on load.

...

If drift was found - user receives response with drift columns.

Example of validate response when schema drift was found:

Expand
Code Block
{
  "status": 0,
  "proc_stats": null,
  "additional_data_type": "string",
  "processing_statuses": [
    [
      "Information",
      "Starting processing an EBS request",
      "eaglepy-i-1000",
      "",
      0,
      "2022-07-07T11:22:16.136+00:00"
    ],
    [
      "Error",
      "Schema validation failed",
      "eaglepy-e-1026",
      "",
      0,
      "2022-07-07T11:22:16.143+00:00"
    ],
    [
      "Information",
      "Executed on svc-eds-5579bc956-fzl48/PID=12/WORKER=/IMPL=CPython",
      "eaglepy-i-1000",
      "",
      0,
      "2022-07-07T11:22:16.143+00:00"
    ]
  ],
  "drift_columns": {
    "_id": "legalentityid",
    "load": {
      "dataframe": {
        "_id": "legalentityid",
        "group_name": "Reference",
        "type": "dftFile",
        "description": "Lei details load",
        "maxrows": -1,
        "source_sink": {
          "sink_type": "ssFileType",
          "sink_params": {},
          "source_descriptor_type": "Mmap",
          "allow_format_discovery": true,
          "source_format_dialect": {
            "lineterminator": "\n",
            "delimiter": ",",
            "headerat": 1,
            "headerlines": 1,
            "dialectname": "csv_char_strip",
            "encode_header": true,
            "force_vocab_stru": true
          }
        },
        "format": "delimited",
        "vocabulary": {
          "newElementChar": {
            "path": "extension_newElementChar",
            "datatype": "string",
            "formatdialect": {
              "length": 16
            }
          },
          "newElementDate": {
            "path": "extension_newElementDate",
            "datatype": "date",
            "formatdialect": {
              "dialect": "YYYY-MM-DD"
            }
          },
          "NewElementNum": {
            "path": "extension_NewElementNum",
            "datatype": "number",
            "formatdialect": {
              "precision": 38,
              "scale": 12
            }
          }
        }
      },
      "interface": {
        "_id": "legalentityid",
        "format": "eaglemlformat",
        "dataset": "legalentityid",
        "dataframe": "legalentityid"
      }
    }
  },
  "worker_id": "svc-eds-5579bc956-fzl48/PID=12/WORKER=/IMPL=CPython"
}

...

Code Block
{
  "resourcename": "legalentityid",
  "dbprovider": "snowflake",
  "schemadriftmode": "ignore"
}

VALIDATE schema drift mode

Schema drift mode VALIDATE is detecting any schema changes for resource on load.

...