diff --git a/skills/explaining-batch-data-transform/assets/sample_bdts/append_and_split.json b/skills/explaining-batch-data-transform/assets/sample_bdts/append_and_split.json new file mode 100644 index 0000000..94eba99 --- /dev/null +++ b/skills/explaining-batch-data-transform/assets/sample_bdts/append_and_split.json @@ -0,0 +1,90 @@ +{ + "version": "66.0", + "nodes": { + "LOAD_WEB_ORDERS": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "WebOrders__dlo", "type": "dataLakeObject"}, + "fields": ["OrderId__c", "Amount__c", "Channel__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "LOAD_STORE_ORDERS": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "StoreOrders__dlo", "type": "dataLakeObject"}, + "fields": ["OrderId__c", "Amount__c", "Channel__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "APPEND_ALL_ORDERS": { + "action": "appendV2", + "sources": ["LOAD_WEB_ORDERS", "LOAD_STORE_ORDERS"], + "parameters": { + "allowImplicitDisjointSchema": false, + "fieldMappings": [ + {"targetField": "OrderId__c", "sources": [{"node": "LOAD_WEB_ORDERS", "field": "OrderId__c"}, {"node": "LOAD_STORE_ORDERS", "field": "OrderId__c"}]}, + {"targetField": "Amount__c", "sources": [{"node": "LOAD_WEB_ORDERS", "field": "Amount__c"}, {"node": "LOAD_STORE_ORDERS", "field": "Amount__c"}]}, + {"targetField": "Channel__c", "sources": [{"node": "LOAD_WEB_ORDERS", "field": "Channel__c"}, {"node": "LOAD_STORE_ORDERS", "field": "Channel__c"}]} + ] + } + }, + "SPLIT_BY_AMOUNT": { + "action": "split", + "sources": ["APPEND_ALL_ORDERS"], + "parameters": { + "branches": [ + {"name": "high_value", "expression": "Amount__c >= 1000"}, + {"name": "low_value", "expression": "Amount__c < 1000"} + ] + } + }, + "OUTPUT_HIGH": { + "action": "outputD360", + "sources": ["SPLIT_BY_AMOUNT"], + "parameters": { + "name": "HighValueOrders__dlm", + "type": "dataModelObject", + "writeMode": "OVERWRITE", + "fieldsMappings": [ + {"sourceField": "OrderId__c", "targetField": "OrderId__c"}, + {"sourceField": "Amount__c", "targetField": "Amount__c"}, + {"sourceField": "Channel__c", "targetField": "Channel__c"} + ] + } + }, + "OUTPUT_LOW": { + "action": "outputD360", + "sources": ["SPLIT_BY_AMOUNT"], + "parameters": { + "name": "LowValueOrders__dlm", + "type": "dataModelObject", + "writeMode": "OVERWRITE", + "fieldsMappings": [ + {"sourceField": "OrderId__c", "targetField": "OrderId__c"}, + {"sourceField": "Amount__c", "targetField": "Amount__c"}, + {"sourceField": "Channel__c", "targetField": "Channel__c"} + ] + } + } + }, + "ui": { + "nodes": { + "LOAD_WEB_ORDERS": {"label": "Web Orders", "type": "LOAD_DATASET", "top": 100, "left": 100}, + "LOAD_STORE_ORDERS": {"label": "Store Orders", "type": "LOAD_DATASET", "top": 260, "left": 100}, + "APPEND_ALL_ORDERS": {"label": "Union", "type": "APPEND", "top": 180, "left": 260}, + "SPLIT_BY_AMOUNT": {"label": "Split by $", "type": "SPLIT", "top": 180, "left": 420}, + "OUTPUT_HIGH": {"label": "High value", "type": "OUTPUT", "top": 100, "left": 580}, + "OUTPUT_LOW": {"label": "Low value", "type": "OUTPUT", "top": 260, "left": 580} + }, + "connectors": [ + {"source": "LOAD_WEB_ORDERS", "target": "APPEND_ALL_ORDERS"}, + {"source": "LOAD_STORE_ORDERS", "target": "APPEND_ALL_ORDERS"}, + {"source": "APPEND_ALL_ORDERS", "target": "SPLIT_BY_AMOUNT"}, + {"source": "SPLIT_BY_AMOUNT", "target": "OUTPUT_HIGH"}, + {"source": "SPLIT_BY_AMOUNT", "target": "OUTPUT_LOW"} + ] + } +} diff --git a/skills/explaining-batch-data-transform/assets/sample_bdts/joins_and_filters.json b/skills/explaining-batch-data-transform/assets/sample_bdts/joins_and_filters.json new file mode 100644 index 0000000..132a355 --- /dev/null +++ b/skills/explaining-batch-data-transform/assets/sample_bdts/joins_and_filters.json @@ -0,0 +1,75 @@ +{ + "version": "66.0", + "nodes": { + "LOAD_ORDERS": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "ssot__SalesOrder__dlm", "type": "dataModelObject"}, + "fields": ["ssot__Id__c", "ssot__AccountId__c", "ssot__Status__c", "ssot__GrandTotalAmount__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "LOAD_ACCOUNTS": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "ssot__Account__dlm", "type": "dataModelObject"}, + "fields": ["ssot__Id__c", "ssot__Name__c", "ssot__Industry__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "FILTER_ACTIVE_ORDERS": { + "action": "filter", + "sources": ["LOAD_ORDERS"], + "parameters": { + "filterExpressions": [ + {"type": "TEXT", "field": "ssot__Status__c", "operator": "EQUAL", "operands": ["Active"]}, + {"type": "NUMBER", "field": "ssot__GrandTotalAmount__c", "operator": "GREATER_THAN", "operands": ["0"]} + ], + "filterBooleanLogic": "1 AND 2" + } + }, + "JOIN_ORDER_ACCOUNT": { + "action": "join", + "sources": ["FILTER_ACTIVE_ORDERS", "LOAD_ACCOUNTS"], + "parameters": { + "joinType": "LEFT_OUTER", + "leftKeys": ["ssot__AccountId__c"], + "rightQualifier": "Account", + "rightKeys": ["ssot__Id__c"] + } + }, + "OUTPUT_ENRICHED": { + "action": "outputD360", + "sources": ["JOIN_ORDER_ACCOUNT"], + "parameters": { + "name": "Order_Enriched__dlm", + "type": "dataModelObject", + "writeMode": "OVERWRITE", + "fieldsMappings": [ + {"sourceField": "ssot__Id__c", "targetField": "OrderId__c"}, + {"sourceField": "ssot__AccountId__c", "targetField": "AccountId__c"}, + {"sourceField": "ssot__GrandTotalAmount__c", "targetField": "Amount__c"}, + {"sourceField": "Account.ssot__Name__c", "targetField": "AccountName__c"}, + {"sourceField": "Account.ssot__Industry__c", "targetField": "Industry__c"} + ] + } + } + }, + "ui": { + "nodes": { + "LOAD_ORDERS": {"label": "Sales Orders", "type": "LOAD_DATASET", "top": 100, "left": 100}, + "LOAD_ACCOUNTS": {"label": "Accounts", "type": "LOAD_DATASET", "top": 260, "left": 100}, + "FILTER_ACTIVE_ORDERS": {"label": "Active, >0", "type": "FILTER", "top": 100, "left": 260}, + "JOIN_ORDER_ACCOUNT": {"label": "Join Account info", "type": "JOIN", "top": 180, "left": 420}, + "OUTPUT_ENRICHED": {"label": "Order Enriched", "type": "OUTPUT", "top": 180, "left": 580} + }, + "connectors": [ + {"source": "LOAD_ORDERS", "target": "FILTER_ACTIVE_ORDERS"}, + {"source": "FILTER_ACTIVE_ORDERS", "target": "JOIN_ORDER_ACCOUNT"}, + {"source": "LOAD_ACCOUNTS", "target": "JOIN_ORDER_ACCOUNT"}, + {"source": "JOIN_ORDER_ACCOUNT", "target": "OUTPUT_ENRICHED"} + ] + } +} diff --git a/skills/explaining-batch-data-transform/assets/sample_bdts/minimal_dmo_to_dmo.json b/skills/explaining-batch-data-transform/assets/sample_bdts/minimal_dmo_to_dmo.json new file mode 100644 index 0000000..4506b24 --- /dev/null +++ b/skills/explaining-batch-data-transform/assets/sample_bdts/minimal_dmo_to_dmo.json @@ -0,0 +1,58 @@ +{ + "version": "66.0", + "nodes": { + "LOAD_ACCOUNT": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "ssot__Account__dlm", "type": "dataModelObject"}, + "fields": ["ssot__Id__c", "ssot__Name__c", "ssot__Industry__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "FORMULA_NAME_UPPER": { + "action": "formula", + "sources": ["LOAD_ACCOUNT"], + "parameters": { + "expressionType": "SQL", + "fields": [ + { + "name": "AccountNameUpper__c", + "label": "Account Name (upper)", + "formulaExpression": "upper(ssot__Name__c)", + "type": "TEXT", + "businessType": "Text", + "precision": 255, + "defaultValue": "" + } + ] + } + }, + "OUTPUT_ACCOUNT_UPPER": { + "action": "outputD360", + "sources": ["FORMULA_NAME_UPPER"], + "parameters": { + "name": "Account_Upper__dlm", + "type": "dataModelObject", + "writeMode": "OVERWRITE", + "fieldsMappings": [ + {"sourceField": "ssot__Id__c", "targetField": "ssot__Id__c"}, + {"sourceField": "AccountNameUpper__c", "targetField": "Name_Upper__c"}, + {"sourceField": "ssot__Industry__c", "targetField": "Industry__c"} + ] + } + } + }, + "ui": { + "nodes": { + "LOAD_ACCOUNT": {"label": "Account", "type": "LOAD_DATASET", "top": 100, "left": 100}, + "FORMULA_NAME_UPPER": {"label": "Uppercase Name", "type": "FORMULA", "top": 100, "left": 260}, + "OUTPUT_ACCOUNT_UPPER": {"label": "Account Upper", "type": "OUTPUT", "top": 100, "left": 420} + }, + "connectors": [ + {"source": "LOAD_ACCOUNT", "target": "FORMULA_NAME_UPPER"}, + {"source": "FORMULA_NAME_UPPER", "target": "OUTPUT_ACCOUNT_UPPER"} + ], + "hiddenColumns": [] + } +} diff --git a/skills/explaining-batch-data-transform/assets/sample_bdts/window_and_aggregate.json b/skills/explaining-batch-data-transform/assets/sample_bdts/window_and_aggregate.json new file mode 100644 index 0000000..ff59375 --- /dev/null +++ b/skills/explaining-batch-data-transform/assets/sample_bdts/window_and_aggregate.json @@ -0,0 +1,74 @@ +{ + "version": "66.0", + "nodes": { + "LOAD_ORDERS": { + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "ssot__SalesOrder__dlm", "type": "dataModelObject"}, + "fields": ["ssot__Id__c", "ssot__AccountId__c", "ssot__CreatedDate__c", "ssot__GrandTotalAmount__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } + }, + "RANK_ORDERS": { + "action": "computeRelative", + "sources": ["LOAD_ORDERS"], + "parameters": { + "partitionBy": ["ssot__AccountId__c"], + "orderBy": [{"fieldName": "ssot__CreatedDate__c", "direction": "ASC"}], + "expressionType": "SQL", + "fields": [ + { + "name": "OrderRank__c", + "label": "Order Rank", + "formulaExpression": "row_number()", + "type": "NUMBER", + "businessType": "Number", + "precision": 18, + "scale": 0, + "defaultValue": "" + } + ] + } + }, + "AGG_BY_ACCOUNT": { + "action": "aggregate", + "sources": ["RANK_ORDERS"], + "parameters": { + "groupings": ["ssot__AccountId__c"], + "aggregations": [ + {"action": "SUM", "name": "TotalAmount__c", "source": "ssot__GrandTotalAmount__c"}, + {"action": "COUNT", "name": "OrderCount__c", "source": "ssot__Id__c"} + ], + "nodeType": "STANDARD" + } + }, + "OUTPUT_SUMMARY": { + "action": "outputD360", + "sources": ["AGG_BY_ACCOUNT"], + "parameters": { + "name": "Account_Summary__dlm", + "type": "dataModelObject", + "writeMode": "OVERWRITE", + "fieldsMappings": [ + {"sourceField": "ssot__AccountId__c", "targetField": "AccountId__c"}, + {"sourceField": "TotalAmount__c", "targetField": "TotalAmount__c"}, + {"sourceField": "OrderCount__c", "targetField": "OrderCount__c"} + ] + } + } + }, + "ui": { + "nodes": { + "LOAD_ORDERS": {"label": "Sales Orders", "type": "LOAD_DATASET", "top": 100, "left": 100}, + "RANK_ORDERS": {"label": "Rank by account", "type": "COMPUTE_RELATIVE", "top": 100, "left": 260}, + "AGG_BY_ACCOUNT": {"label": "Totals per account", "type": "AGGREGATE", "top": 100, "left": 420}, + "OUTPUT_SUMMARY": {"label": "Account Summary", "type": "OUTPUT", "top": 100, "left": 580} + }, + "connectors": [ + {"source": "LOAD_ORDERS", "target": "RANK_ORDERS"}, + {"source": "RANK_ORDERS", "target": "AGG_BY_ACCOUNT"}, + {"source": "AGG_BY_ACCOUNT", "target": "OUTPUT_SUMMARY"} + ] + } +} diff --git a/skills/explaining-batch-data-transform/references/bdt-function-catalog.md b/skills/explaining-batch-data-transform/references/bdt-function-catalog.md new file mode 100644 index 0000000..4f59160 --- /dev/null +++ b/skills/explaining-batch-data-transform/references/bdt-function-catalog.md @@ -0,0 +1,124 @@ +# SFSQL Function Catalog — for `formula` reasoning + +> **Last synced:** 2026-04-23 from the Data Processing Engine PDF (2025-06-24 version) and +> the BDT help DITA XMLs (`c360_a_batch_transform_numeric.xml`, `_string.xml`, +> `_date_functions.xml`, `_boolean_functions.xml`, `_multivalue_functions.xml`, +> `_additionalfunctions.xml`). Consult this file before interpreting any `formulaExpression`. + +## How formulas appear in BDT JSON + +```jsonc +"action": "formula", +"parameters": { + "expressionType": "SQL", // or "DCSQL" + "fields": [ + { + "name": "OutputField__c", + "label": "Output Label", + "type": "TEXT", // TEXT | NUMBER | BOOLEAN | DATE_ONLY | DATETIME + "businessType": "Text", // user-visible type name + "formulaExpression": "upper(coalesce(SourceField__c, ''))", + "precision": 255, + "scale": 0, // only for NUMBER + "defaultValue": "" + } + ] +} +``` + +## Functions by family + +### Math / Numeric + +| Function | Signature | Purpose | +|---|---|---| +| `abs(n)` | number → number | Absolute value (strips sign). | +| `ceiling(n)` | number → number | Round up, away from zero for negatives. | +| `floor(n)` | number → number | Round down, toward zero for negatives (or toward the next integer up for negatives). | +| `exp(n)` | number → number | e raised to n. | +| `log(base, n)` | number → number | Logarithm of n in the given base. | +| `max(a, b, …)` | number → number | Largest value. | +| `min(a, b, …)` | number → number | Smallest value. | +| `mod(a, b)` | number → number | Remainder after dividing a by b. | +| `power(a, b)` | number → number | a raised to the power b. | +| `round(n, digits)` | number → number | Round to `digits` places. | +| `sqrt(n)` | number → number | Positive square root. | +| `trunc(n, digits)` | number → number | Truncate to `digits` places (doesn't round). | + +### String + +| Function | Signature | Purpose | +|---|---|---| +| `begins(s, prefix)` | text → boolean | True if s starts with prefix. | +| `concat(a, b, …)` | text → text | Concatenate. | +| `contains(haystack, needle)` | text → boolean | True if haystack contains needle. | +| `ends(s, suffix)` | text → boolean | True if s ends with suffix. | +| `length(s)` | text → number | Character count. | +| `lower(s)` | text → text | Lowercase (locale-aware if a locale is provided). | +| `ltrim(s)` / `ltrim(s, substring)` | text → text | Remove leading whitespace (or a specific substring). | +| `rtrim(s)` / `rtrim(s, substring)` | text → text | Remove trailing whitespace (or substring). | +| `substitute(s, old, new)` | text → text | Replace old with new in s. | +| `substr(s, start, length)` | text → text | Extract a substring. | +| `text(n)` | number → text | Convert a number to its text form. | +| `trim(s)` / `trim(s, substring)` | text → text | Remove leading + trailing whitespace or substring. | +| `upper(s)` | text → text | Uppercase. | +| `uuid()` | → text | Newly generated unique ID. | +| `value(s)` | text → number | Parse a text representation of a number into a numeric value. | + +### Date + +| Function | Signature | Purpose | +|---|---|---| +| `adddays(date, n)` | date → date | Add n days. | +| `addmonths(date, n)` | date → date | Add n months. | +| `datediff(start, end)` | date × date → number | Days between two dates. | +| `datetimevalue(text_or_date)` | → datetime | GMT/UTC date+time value. | +| `datevalue(text_or_datetime)` | → date | Extract the date part. | +| `day(date)` | date → number | 1-31. | +| `monthdiff(start, end)` | date × date → number | Months between two dates. | +| `now()` | → datetime | Current moment (UTC). | +| `today()` | → date | Current date. | +| `weekday(date)` | date → number | 1=Sunday, 2=Monday, … 7=Saturday. | + +### Boolean / Logical + +| Function | Signature | Purpose | +|---|---|---| +| `and(a, b, …)` | bool → bool | True when all are true. | +| `or(a, b, …)` | bool → bool | True when any is true. | +| `if(cond, then_val, else_val)` | → any | Ternary. | +| `case when then else end` | — | Multi-branch (SQL CASE). | +| `blankvalue(expr, substitute)` | → any | Substitute when expr is blank. | +| `isblank(expr)` | → bool | True when expr is blank. | +| `isnull(expr)` | → bool | True when expr is null. | +| `nullvalue(expr, substitute)` | → any | Substitute when expr is null; returns expr otherwise. | + +### Multivalue + +| Function | Signature | Purpose | +|---|---|---| +| `sequence(start, end, step?)` | → array | Array of numbers/dates between start and end (step defaults to 1 / 1 day). | +| `explode(array)` | → row per element | Convert multivalue data into one row per element. **Cannot be nested inside another function.** | + +### Additional / Null-handling (commonly seen) + +| Function | Signature | Purpose | +|---|---|---| +| `coalesce(a, b, …)` | → any | First non-null argument (SQL standard). | + +## Patterns to recognize in narration + +- **`coalesce(X, Y)`** — "use X when present, otherwise Y." Very common for default-value + handling. +- **`case when RANK = 1 then X else 0 end`** — the "first-occurrence extract" idiom paired + with a `computeRelative` `row_number()` node. +- **`case when FLAG = 'Y' then 'Y' else 'N' end`** — boolean-like text normalization. +- **`concat(coalesce(A, ''), coalesce(B, ''))`** — safe string concatenation that defaults + NULLs to empty strings (producing a synthetic composite key). + +## Sources + +- Data Processing Engine reference PDF (captured at `research/bdt-doc-10-data_processing_engine_6-24-2025.pdf.md`) +- BDT help XML: `c360_a_batch_transform_numeric.xml`, `_string.xml`, `_date_functions.xml`, + `_boolean_functions.xml`, `_multivalue_functions.xml`, `_additionalfunctions.xml` (captured + at `research/help-xml/`). diff --git a/skills/explaining-batch-data-transform/references/bdt-node-catalog.md b/skills/explaining-batch-data-transform/references/bdt-node-catalog.md new file mode 100644 index 0000000..1f3312c --- /dev/null +++ b/skills/explaining-batch-data-transform/references/bdt-node-catalog.md @@ -0,0 +1,742 @@ +# BDT Node Catalog — per-action reference (on-demand) + +> **Source of truth:** `research/bdt-schema-canonical.md`, extracted from the BDT Connect +> API Java sources (`cdp-connect-api` module, release 264) on 2026-04-23. The entries +> below are curated to answer the skill's job — explaining BDT JSON. For the full +> extraction (all fields, minVersion annotations, polymorphism map), see the research doc. + +Consult this file before explaining any node type. Each section covers: +- **JSON action name** and UI name(s). +- **Purpose** in plain English. +- **Key parameters** with types (and enum values where applicable). +- **How it affects lineage** — rename? add? drop? change row cardinality? +- **Common gotchas**. +- **Example JSON snippet**. +- **Source citation** back to the canonical Java input-rep class. + +--- + +## `action: "load"` — UI: "Load" / "Data Source" + +**Purpose.** Reads rows from a DMO or DLO into the pipeline. Every BDT has at least one +load node (they are the graph roots — `sources: []`). + +**Key parameters** (class `LoadParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `dataset` | `LoadDatasetInputRepresentation` | `{ name: string, type: "dataModelObject" \| "dataLakeObject" }` | +| `fields` | `string[]` | Field names to pull from the source. Not pulling a field means it's unavailable downstream. | +| `sampleDetails` | `{ type: "TopN" \| "Custom" \| "Unique", sortBy: string[] }` | Editor-only sampling behavior; doesn't affect runtime. | + +**Lineage effect.** Defines the *initial* set of field names available downstream. No row +cardinality change (all rows are loaded). + +**Gotchas.** +- A field referenced by a downstream node must appear in some load's `fields` array, **or** + be generated later by a formula/aggregate. If you can't trace a field to a load or a + derivation, the BDT is broken. +- `dataset.type` was historically called `dataLakeObject` in some older dialects; per + canonical schema both `dataModelObject` and `dataLakeObject` are valid. +- `sampleDetails.sortBy` is a list of strings — may be empty. + +**Example.** +```jsonc +{ + "action": "load", + "sources": [], + "parameters": { + "dataset": {"name": "ssot__SalesOrder__dlm", "type": "dataModelObject"}, + "fields": ["ssot__Id__c", "ssot__AccountId__c", "ssot__GrandTotalAmount__c"], + "sampleDetails": {"type": "TopN", "sortBy": []} + } +} +``` + +**Source.** `LoadNodeInputRepresentation` + `LoadParametersInputRepresentation` + +`LoadDatasetInputRepresentation`. + +--- + +## `action: "join"` — UI: "Join" + +**Purpose.** Joins two upstream streams by key. Always has exactly two `sources`; field names +on the right-hand side get a qualifier prefix so they don't collide with left-hand names. + +**Key parameters** (class `JoinParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `joinType` | enum `JoinType`: `INNER`, `OUTER`, `LEFT_OUTER`, `RIGHT_OUTER`, `LOOKUP`, `MULTI_VALUE_LOOKUP`, `CROSS` | | +| `leftKeys` | `string[]` | Join keys on the left (first) source. | +| `rightKeys` | `string[]` | Join keys on the right (second) source. | +| `leftQualifier` | `string` (optional) | Prefix for left-side field names in the output. Often omitted. | +| `rightQualifier` | `string` (required) | Prefix for right-side fields (e.g., `SalesOrder` → right-side field `ssot__Id__c` becomes `SalesOrder.ssot__Id__c`). | + +**Optional node-level** `schema.slice`: +- `{ mode: "DROP" | "SELECT", fields: string[], ignoreMissingFields: bool }` +- Used to trim unwanted fields from the joined output. + +**Lineage effect.** Combines two field sets. Fields from the right side get renamed with +`rightQualifier` prefix. Row cardinality depends on join type: +- `INNER` — only matched pairs. +- `LEFT_OUTER` — all left rows + matching right. +- `RIGHT_OUTER` — all right rows + matching left. +- `OUTER` — all rows from both sides. +- `LOOKUP` — 1:1 left-side preserving. +- `MULTI_VALUE_LOOKUP` — left-side preserving; multi-valued right. +- `CROSS` — Cartesian product; every left paired with every right. + +**Gotchas.** +- Data types of joined keys should match. Type mismatches cause silent no-match or errors + depending on pair. +- `rightQualifier` is the *only* way to disambiguate same-named fields from two sources. + Always surface it when narrating. +- `schema.slice` on a join is a post-join projection, not a pre-join filter. + +**Example.** +```jsonc +{ + "action": "join", + "sources": ["JOIN7", "FILTER4"], + "parameters": { + "joinType": "LEFT_OUTER", + "leftKeys": ["ssot__Id__c"], + "rightQualifier": "SalesOrder", + "rightKeys": ["ssot__SalesOrderId__c"] + }, + "schema": { + "slice": { + "mode": "DROP", + "ignoreMissingFields": true, + "fields": ["SalesOrder.ssot__InternalOrganizationId__c"] + } + } +} +``` + +**Source.** `JoinNodeInputRepresentation` + `JoinParametersInputRepresentation`. + +--- + +## `action: "filter"` — UI: "Filter" + +**Purpose.** Keep only rows satisfying filter criteria combined by boolean logic. + +**Key parameters** (class `FilterParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `filterExpressions` | `FilterExpression[]` | Each expression is `{ field, operator, type, operands[] }`. | +| `filterBooleanLogic` | `string` | A formula like `"1 AND (2 OR 3)"` indexed 1..N into `filterExpressions`. If omitted, default is all AND. | + +**FilterExpression fields** (class `FilterExpressionInputRepresentation`): +- `field` — the field the filter examines. +- `operator` — `string` (no typed enum in the canonical input-rep; free-form in the API). + Commonly observed values: `EQUAL`, `NOT_EQUAL`, `GREATER_THAN`, `LESS_THAN`, + `GREATER_OR_EQUAL`, `LESS_OR_EQUAL`, `IN_RANGE`, `LIKE`, `IS_NULL`, `IS_NOT_NULL`. +- `type` — enum `DataType` (same as elsewhere in BDT): `TEXT`, `NUMBER`, `BOOLEAN`, + `DATE_ONLY`, `DATETIME`. +- `operands` — array of operand values (strings in JSON; interpreted per `type`). + +**Lineage effect.** Does not change field names; only reduces rows. + +**Gotchas.** +- `filterBooleanLogic` operands are **1-indexed** into `filterExpressions`. Be careful when + narrating which expression is "expression 1". +- If a referenced field is dropped upstream, the filter becomes invalid at runtime. + +**Example.** +```jsonc +{ + "action": "filter", + "sources": ["LOAD_DATASET0"], + "parameters": { + "filterExpressions": [ + {"type": "TEXT", "field": "MobilePhone_Formatted_Flag__c", "operator": "EQUAL", "operands": ["Y"]}, + {"type": "TEXT", "field": "IsActive__c", "operator": "EQUAL", "operands": ["true"]} + ], + "filterBooleanLogic": "1 AND 2" + } +} +``` + +**Source.** `FilterNodeInputRepresentation` + `FilterParametersInputRepresentation` + +`FilterExpressionInputRepresentation`. + +--- + +## `action: "sqlFilter"` — UI: "SQL Filter" + +**Purpose.** A filter whose predicate is a raw SQL expression — more expressive than the +structured `filter` node, at the cost of being harder to validate statically. + +**Key parameters** (class `SqlFilterParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `sqlFilterExpression` | `string` | A SQL WHERE-clause-like predicate referring to fields by name. | + +**Lineage effect.** Same as `filter` — reduces rows only. + +**Example.** +```jsonc +{ + "action": "sqlFilter", + "sources": ["JOIN1"], + "parameters": { + "sqlFilterExpression": "ssot__CreatedDate__c >= current_date() - interval '30' day" + } +} +``` + +**Source.** `SqlFilterNodeInputRepresentation` + `SqlFilterParametersInputRepresentation`. + +--- + +## `action: "formula"` — UI: "Formula" + +**Purpose.** Add one or more derived columns via per-row SQL formulas. No window / cross-row +semantics — for that, see `computeRelative`. + +**Key parameters** (class `FormulaParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `expressionType` | enum `FormulaExpressionType`: `SQL`, `DCSQL` | | +| `fields` | `SqlFormulaFieldInputRepresentation[]` | One entry per derived field. | + +**Each field** carries: +- `name` — the output field name. +- `label` — user-visible label. +- `formulaExpression` — SFSQL (or DCSQL) expression. See `bdt-function-catalog.md`. +- `type` — enum `DataType`: `TEXT`, `NUMBER`, `BOOLEAN`, `DATE_ONLY`, `DATETIME`. +- `businessType` — a user-facing business-semantic type name. Enum `BusinessTypeEnum` — + canonical values: + - `"TEXT"`, `"NUMBER"`, `"BOOLEAN"` + - `"EMAIL"`, `"PHONE"`, `"URL"` — text-valued with semantic meaning + - `"PERCENT"`, `"CURRENCY"` — number-valued with semantic meaning + - `"DATE"` — semantically a date, stored as datetime at the underlying `type` level + - `"DATE_ONLY"` — date-only value (no time component) + - `"DATETIME"` — full datetime + + Note: `businessType` values map to underlying `type` (`DataType`) values. E.g., + `businessType: "PERCENT"` is stored as `type: "NUMBER"`; `businessType: "DATE"` is stored + as `type: "DATETIME"`; `businessType: "EMAIL"` is stored as `type: "TEXT"`. The canonical + mapping lives in `BusinessTypeEnum.java`. +- `precision` — integer precision (default 10 for numbers; characters for text). +- `scale` — decimal places; only for NUMBER. +- `defaultValue` — value when the expression yields NULL. + +**Lineage effect.** Adds new columns to the downstream row stream. Original columns pass +through unchanged unless dropped later by a `schema` node. Row cardinality unchanged. + +**Gotchas.** +- If a field referenced in `formulaExpression` is removed upstream, the formula fails at + runtime. +- **`type` and `businessType`** — both use UPPER wire form per canonical enums + (`type: "NUMBER"`, `businessType: "NUMBER"`). Some user-authored BDT JSON + may show capitalized forms (`"Number"`) — the runtime is case-insensitive + per `BusinessTypeEnum.valueOfInternal()`, but the canonical wire form is + UPPER. +- Concrete sub-classes exist for typed fields (`SqlFormulaNumericFieldInputRepresentation`, + etc.) but the JSON shape is the same. + +**Example.** +```jsonc +{ + "action": "formula", + "sources": ["JOIN5"], + "parameters": { + "expressionType": "SQL", + "fields": [ + { + "name": "ssot__SalesOrderProductConcat__c", + "label": "SalesOrderProductConcat", + "formulaExpression": "concat(coalesce(\"SalesOrder.ssot__Id__c\",'NULL'),coalesce(\"SalesOrder.ssot__Id__c\",''))", + "type": "TEXT", + "businessType": "TEXT", + "precision": 60, + "defaultValue": "" + } + ] + } +} +``` + +**Source.** `FormulaNodeInputRepresentation` + `FormulaParametersInputRepresentation` + +`SqlFormulaFieldInputRepresentation`. + +--- + +## `action: "computeRelative"` — UI: "Window Transform" + +**Purpose.** A formula that evaluates a **window function** over partitioned, ordered rows. +Used for ranking, lead/lag, running totals, etc. + +**Key parameters** (class `ComputeRelativeParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `partitionBy` | `string[]` | Column(s) to partition rows by. Empty = whole stream. | +| `orderBy` | `ComputeRelativeSortParametersInputRepresentation[]` | Each `{ fieldName, direction: "ASC" \| "DESC" }`. | +| `expressionType` | enum `FormulaExpressionType`: `SQL`, `DCSQL` | | +| `fields` | `SqlFormulaFieldInputRepresentation[]` | Same shape as `formula` fields. | + +**Lineage effect.** Adds one or more columns. Original columns pass through. Row cardinality +unchanged. + +**Gotchas.** +- The documentation states "A formula can include only one Compute Relative function" — if + you see multiple compute-relative calls in one expression, flag as unusual. +- If `partitionBy` is empty, the function runs over the whole stream (one big window). +- `orderBy` is required for order-dependent functions (`row_number`, `rank`, `lag`, etc.). + Without it, results are non-deterministic. + +**Example.** +```jsonc +{ + "action": "computeRelative", + "sources": ["LOAD_ORDERS"], + "parameters": { + "partitionBy": ["ssot__AccountId__c"], + "orderBy": [{"fieldName": "ssot__CreatedDate__c", "direction": "ASC"}], + "expressionType": "SQL", + "fields": [ + { + "name": "OrderRank__c", + "label": "Order Rank", + "formulaExpression": "row_number()", + "type": "NUMBER", + "businessType": "NUMBER", + "precision": 18, + "scale": 0, + "defaultValue": "" + } + ] + } +} +``` + +**Source.** `ComputeRelativeNodeInputRepresentation` + +`ComputeRelativeParametersInputRepresentation` + +`ComputeRelativeSortParametersInputRepresentation`. For the function catalog, see +`bdt-window-functions.md`. + +--- + +## `action: "aggregate"` — UI: "Aggregate" / "Group and Aggregate" + +**Purpose.** Group-by aggregation. Also supports a hierarchical mode for parent/child aggregation. + +**Key parameters** (class `AggregateParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `groupings` | `string[]` | Group-by field names. **Not** `groupBy`. | +| `aggregations` | `AggregateInputRepresentation[]` | Each `{ action: AggregateType, name, source, label? }`. | +| `nodeType` | enum `AggregateNodeEnum`: `STANDARD`, `HIERARCHICAL` | | +| `selfField`, `parentField`, `percentageField` | `string` (hierarchical-only) | | +| `pivot_v2` | `PivotV2InputRepresentation` (optional) | Advanced; pivot in the same node. | + +**Aggregation functions** (enum `AggregateType`): +`UNIQUE`, `SUM`, `AVG`, `COUNT`, `MAX`, `MIN`, `MEDIAN`, `STDDEVP`, `STDDEV`, `VARP`, `VAR`. + +**Lineage effect.** +- Output columns = `groupings` (pass-through) + each `aggregation.name` (new derived column). +- Row cardinality: one output row per unique combination of `groupings`. + +**Gotchas.** +- Group-by column is called `groupings` not `groupBy` in the JSON. +- `aggregation.source` is the field being aggregated; `aggregation.name` is the output column name. +- In `HIERARCHICAL` mode, the three `*Field` parameters carry parent-child semantics; the + aggregations roll up across the hierarchy. + +**Example.** +```jsonc +{ + "action": "aggregate", + "sources": ["FILTER0"], + "parameters": { + "groupings": ["ssot__AccountId__c"], + "aggregations": [ + {"action": "SUM", "name": "TotalAmount__c", "source": "ssot__GrandTotalAmount__c"}, + {"action": "COUNT", "name": "OrderCount__c", "source": "ssot__Id__c"} + ], + "nodeType": "STANDARD" + } +} +``` + +**Source.** `AggregateNodeInputRepresentation` + `AggregateParametersInputRepresentation` + +`AggregateInputRepresentation`. + +--- + +## `action: "schema"` — UI: "Edit Attributes" / "Drop Fields" + +**Purpose.** Modify column-level schema: rename columns, change properties, or slice the set +of columns. + +**Key parameters** (class `SchemaParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `fields` | `SchemaFieldParametersInputRepresentation[]` | Per-field: `{ name, newProperties: { name?, label? } }`. | +| `slice` | `SchemaSliceInputRepresentation` (optional) | `{ mode: "DROP" \| "SELECT", fields: string[], ignoreMissingFields: bool }`. | + +**Slice semantics.** +- `mode: DROP` — remove the listed fields from the output. +- `mode: SELECT` — keep only the listed fields. +- `ignoreMissingFields: true` means listed fields that aren't present are silently ignored. + Without this flag, missing fields would error. + +**Lineage effect.** Schema-only: renames or drops columns. Row cardinality unchanged. + +**Gotchas.** +- `ignoreMissingFields: true` plus a typo = silent failure. When explaining a schema node + that uses it, flag the risk ("these field names are silently skipped if missing"). +- `fields[].newProperties.name` is where a rename lands; the original `name` is the key. + +**Example (drop).** +```jsonc +{ + "action": "schema", + "sources": ["FORMULA43"], + "parameters": { + "slice": { + "mode": "DROP", + "ignoreMissingFields": true, + "fields": ["ssot__SalesOrderProductConcat__c", "FirstPurchase__c"] + } + } +} +``` + +**Example (rename).** +```jsonc +{ + "action": "schema", + "sources": ["AGGREGATE0"], + "parameters": { + "fields": [ + {"name": "sum_amount", "newProperties": {"name": "TotalAmount__c", "label": "Total Amount"}} + ] + } +} +``` + +**Source.** `SchemaNodeInputRepresentation` + `SchemaParametersInputRepresentation` + +`SchemaFieldParametersInputRepresentation` + `SchemaSliceInputRepresentation`. + +--- + +## `action: "outputD360"` — UI: "Output" / "Writeback" + +**Purpose.** Writes the row stream to a target DMO or DLO. Every BDT has at least one +outputD360 node — these are the graph sinks. + +**Key parameters** (class `OutputD360ParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `name` | `string` | Target object's API name (e.g., `Account_Upper__dlm`). | +| `type` | enum `D360OutputTypeEnum`: `dataModelObject`, `dataLakeObject` | | +| `writeMode` | enum `WriteModeEnum`: `APPEND`, `MERGE`, `OVERWRITE`, `MERGE_UPSERT_DELETE`, `DELETE_ONLY` | Write semantics. | +| `fieldsMappings` | `OutputD360FieldsMappingInputRepresentation[]` | Each `{ sourceField, targetField }` pair. | +| `dedupOrder` | `SortSpecificationRepresentation[]` | Tiebreaker for deduplicating records with the same primary key. | +| `streaming` | `StreamingParametersInputRepresentation` (optional) | Streaming-specific; not relevant for BDT. | + +**Write modes.** +- `APPEND` — add rows; no primary-key checks. +- `OVERWRITE` — replace the target with the dataset. +- `MERGE` — merge on PK: update matching rows (only for columns present in the input), insert + new rows. +- `MERGE_UPSERT_DELETE` — merge with per-row UPSERT/DELETE markers. +- `DELETE_ONLY` — delete matching rows. + +The underlying DaaS library supports more modes (`OVERWRITE_PARTITIONS`, +`OVERWRITE_PARTITION_FILTER`, `SECONDARY_INDEX_INCREMENTAL_WRITE`), but the BDT Connect API +exposes only the five above. + +**Lineage effect.** Terminal node. Maps source-stream fields to target-object fields; any +unmapped source field is discarded. + +**Gotchas.** +- `OUTPUT0 nodes must have DataModelObject type` is a known restriction in some frameworks + (data-kit templates). If a BDT has `type: dataLakeObject` and is failing to install in a + template context, that may be the cause. +- If multiple input rows have the same primary key, `dedupOrder` decides which wins. +- Fields not listed in `fieldsMappings` are not written. + +**Example.** +```jsonc +{ + "action": "outputD360", + "sources": ["DROP_FIELDS0"], + "parameters": { + "name": "Kohler_Internal_Users__dlm", + "type": "dataModelObject", + "writeMode": "MERGE", + "fieldsMappings": [ + {"sourceField": "Formatted_MobilePhone__c", "targetField": "Formatted_MobilePhone__c"}, + {"sourceField": "Kohler_Internal_User_Flag", "targetField": "Kohler_Internal_User_Flag__c"} + ] + } +} +``` + +**Source.** `OutputD360NodeInputRepresentation` + `OutputD360ParametersInputRepresentation` + +`OutputD360FieldsMappingInputRepresentation`. + +--- + +## `action: "appendV2"` — UI: "Append" + +**Purpose.** Union rows from two or more upstream streams into one output. + +**Key parameters** (class `AppendParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `columnMappings` | `Map` | Map of source-stream-node-name → column mapping. | +| `fieldMappings` | `AppendMappingInputRepresentation[]` | Explicit per-field mappings. | +| `allowImplicitDisjointSchema` | `boolean` | When true, sources with different fields are merged; missing fields become NULL. | + +**Lineage effect.** +- Rows: union of input streams. +- Columns: the merged schema (union if `allowImplicitDisjointSchema`; otherwise intersection). + +**Gotchas.** +- Source nodes must have the same column count *and* matching column names (in order), unless + `allowImplicitDisjointSchema: true`. +- Append can accept up to 200 fields total from its sources (per help docs). + +**Example.** +```jsonc +{ + "action": "appendV2", + "sources": ["LOAD_A", "LOAD_B"], + "parameters": { + "fieldMappings": [ + {"targetField": "Id__c", "sources": [{"node": "LOAD_A", "field": "ssot__Id__c"}, + {"node": "LOAD_B", "field": "ssot__Id__c"}]} + ] + } +} +``` + +**Source.** `AppendV2NodeInputRepresentation` + `AppendParametersInputRepresentation` + +`AppendMappingInputRepresentation`. + +--- + +## `action: "split"` — UI: "Split" + +**Purpose.** Route rows into multiple downstream branches based on per-branch predicates. + +**Key parameters** (class `SplitParametersInputRepresentation`): + +| Param | Type | Notes | +|---|---|---| +| `branches` | array of `{ name, expression }` | Per-branch routing predicate. | + +**Lineage effect.** +- Rows: partitioned across branches (each branch exposes the rows matching its predicate). +- Columns: same as input. + +**Gotchas.** +- A split node has one upstream source but *multiple* downstream consumers (each branch is a + logical output). When narrating lineage, mention which branch feeds which downstream node. + +**Source.** `SplitNodeInputRepresentation` + `SplitParametersInputRepresentation`. + +--- + +## `action: "flatten"` / `"flattenJson"` — UI: "Flatten" / "Flatten JSON" + +**Purpose.** +- `flatten` — flatten a nested (array-valued or structured) field into additional rows. +- `flattenJson` — flatten a JSON string field by parsing it and emitting its fields as + columns (or extracting array elements as rows via a subsequent `extractTable`). + +**Key parameters** (`FlattenParametersInputRepresentation`, +`FlattenJsonParametersInputRepresentation`): + +- `fields` — list of `FlattenFieldInputRepresentation { name, attributePath?, label? }`. +- For JSON, a schema description may be embedded. + +**Lineage effect.** May increase rows (array-explode) or add columns (object-flatten). + +**Source.** `FlattenNodeInputRepresentation`, `FlattenJsonNodeInputRepresentation`. + +--- + +## `action: "extractGrains"` — UI: "Extract Grains" + +**Purpose.** Expand rows across time grains — e.g., a date column + list of grains produces +one row per source row per grain. + +**Key parameters** (class `ExtractGrainParametersInputRepresentation`): + +- `grainExtractions` — each `{ source, targets: [{ name, label, grainType }] }`. +- `dateConfigurationName` — which date configuration (fiscal calendar, etc.) to use. + +**Valid `grainType` values** (enum `DateGrain`): +`YEAR, QUARTER, MONTH, WEEK, DAY, HOUR, MINUTE, SECOND, DAY_EPOCH, SEC_EPOCH, FISCAL_YEAR, +FISCAL_QUARTER, FISCAL_MONTH, FISCAL_WEEK`. + +**Lineage effect.** Usually adds one column per grain type; may or may not multiply rows +depending on configuration. + +**Source.** `ExtractGrainNodeInputRepresentation` + `ExtractGrainParametersInputRepresentation`. + +--- + +## `action: "extractTable"` — UI: "Extract Table" + +**Purpose.** Pairs with `flattenJson`: extract a named table (from a JSON array) as a +separate output stream. + +**Key parameters.** See `ExtractTableParametersInputRepresentation`. + +**Source.** `ExtractTableNodeInputRepresentation`. + +--- + +## `action: "typeCast"` — UI: "Type Cast" + +**Purpose.** Cast one or more fields to new types. + +**Key parameters.** See `TypecastParametersInputRepresentation`, +`SchemaTypePropertiesCastInputRepresentation`. + +**Lineage effect.** Schema-only; row cardinality unchanged. + +--- + +## `action: "bucket"` — UI: "Bucket Date/Dimension/Measure" + +**Purpose.** Assign bucket labels to values of a source field, per a bucket setup. + +**Key parameters** (class `BucketParametersInputRepresentation`): + +- `fields` — list of `BucketFieldInputRepresentation`, polymorphic on the field type: + - Boolean: `BucketBooleanFieldInputRepresentation` + - DateOnly: `BucketDateOnlyFieldInputRepresentation` + - DateTime: `BucketDateTimeFieldInputRepresentation` + - Dimension: `BucketDimensionFieldInputRepresentation` + - Measure: `BucketMeasureFieldInputRepresentation` + +Each sub-class includes a `setup` object describing the buckets (ranges, algorithms, labels). + +**Algorithm type** (enum `BucketAlgorithmType`): `TYPOGRAPHIC_CLUSTERING` (and potentially +others — verify against sample if needed). + +**Lineage effect.** Adds a derived bucket-label column. Row cardinality unchanged. + +--- + +## `action: "formatDate"` — UI: "Format Dates" + +**Purpose.** Reformat date fields — e.g., parse a custom string format into a date type, +or produce a formatted text representation. + +**Key parameters.** See `FormatDateParametersInputRepresentation`, +`FormatDatePatternInputRepresentation`. + +--- + +## `action: "update"` — UI: "Update" + +**Purpose.** Update records in place in the pipeline (used in specific update workflows). + +**Key parameters.** See `UpdateParametersInputRepresentation`. + +--- + +## `action: "extension"` / `"extensionFunction"` — UI: custom extensions + +**Purpose.** Run a custom extension node / function registered with Data Cloud. + +**Key parameters.** See `ExtensionParametersInputRepresentation`, +`ExtensionFunctionParametersInputRepresentation`, +`ExtensionFunctionOutputFieldInputRepresentation`. + +**Gotcha.** Extensions are user-defined; explanation must rely on parameter content since +the semantics are defined outside the BDT spec. + +--- + +## `action: "cdpPredict"` — UI: "Predict" + +**Purpose.** Apply a prediction model (CDP Predict / Einstein). + +**Key parameters.** See `CdpPredictNodeInputRepresentation` + +`CdpPredictParametersInputRepresentation` + `PredictionFieldInputRepresentation` + +`PredictSourceInputRepresentation`. + +--- + +## `action: "jsonAggregate"` — UI: "JSON Aggregate" + +**Purpose.** Aggregate JSON-valued fields into structured output. + +**Key parameters.** See the `JsonAggregateEnum` enum and the relevant input representations +(currently sparsely documented — inspect raw parameters when encountered). + +--- + +## `action: "save"` — UI: "Save" + +**Purpose.** Save an intermediate result to a checkpoint (not a final output). Less common. + +--- + +## `action: "recommendation"` — UI: "Recommendation" + +**Purpose.** Apply a recommendation model (product recommendations). + +**Key parameters.** See `PredictionContributorInputRepresentation`. + +--- + +## Common `schema.slice` block (can appear on most node types) + +Many nodes accept an optional top-level `schema.slice` block to trim fields at the node +boundary. Its shape is always the same: + +```jsonc +"schema": { + "slice": { + "mode": "DROP" | "SELECT", + "fields": ["Field1", "Field2"], + "ignoreMissingFields": true + } +} +``` + +- `DROP` — remove the listed fields. +- `SELECT` — keep only the listed fields. +- `ignoreMissingFields: true` — silent skip when a listed field doesn't exist. + +--- + +## Polymorphism map (for JSON parsers) + +Several input reps are polymorphic; the discriminator is a JSON property: + +- `AbstractBucketAlgorithmInputRepresentation` discriminated by `"type"`: + - `"TYPOGRAPHIC_CLUSTERING"` → `TypographicClusterInputRepresentation`. + +Other polymorphic classes (e.g., `SqlFormulaFieldInputRepresentation` → numeric / text / +boolean / date variants) are resolved at deserialization based on `type`. When narrating, +the field-level JSON usually carries the concrete fields directly — no special handling +needed. + +## Source citations + +Every section above traces back to one or more classes in the BDT Connect API +`cdp-connect-api` module, packages `sfdc.cdp.connect.api.{input,enums}.datatransform`. +See `research/bdt-schema-canonical.md` for the full machine-extracted schema plus +version-drift verification across releases 260, 262, and 264. diff --git a/skills/explaining-batch-data-transform/references/bdt-reference.md b/skills/explaining-batch-data-transform/references/bdt-reference.md new file mode 100644 index 0000000..d00a50e --- /dev/null +++ b/skills/explaining-batch-data-transform/references/bdt-reference.md @@ -0,0 +1,179 @@ +# BDT Reference — Overview (always loaded) + +> **Last synced:** 2026-04-23 from canonical BDT Connect API Java sources +> (`cdp-connect-api` module, `sfdc.cdp.connect.api.{input,enums}.datatransform` packages, +> release 264). Cross-verified identical across releases 260, 262, and 264. See +> `research/bdt-schema-canonical.md` for the full machine-extracted schema. + +## What a BDT is + +**Batch Data Transform (BDT)** — a repeatable, scheduled or on-demand data pipeline inside +Salesforce Data Cloud. It reads from one or more source objects, applies a DAG of +transformations (joins, filters, formulas, aggregations, etc.), and writes one or more +outputs. BDTs run over batches; their streaming counterpart is **SDT** (Streaming Data +Transform). + +- Salesforce object type: `MktDataTransform`. +- Typical export: a JSON file from the BDT viewer, or via Workbench + `/ssot/data-transforms?htmlEncode=false`. +- Underlying runtime: the Data Processing Engine (DPE) / DCSQL. + +## Top-level JSON shape + +A BDT JSON can arrive in **three** shapes. The skill's parser accepts all three +transparently (since commit `ee2485c`); the inner node graph is identical in every case. + +### 1. Editor export (the default "download" shape) + +What you get when you export a BDT from the BDT editor UI, or fetch it via +`/ssot/data-transforms?htmlEncode=false` on Workbench. The outer object is the +**definition** itself, with no wrapper metadata: + +```jsonc +{ + "version": "66.0", // Schema version string + "nodes": { /* NodeName → Node */ }, + "ui": { // Optional: layout hints + human-readable labels + "nodes": { /* NodeName → { label, description, type, top, left } */ }, + "connectors": [ /* { source, target } edges */ ], + "hiddenColumns": [] + } +} +``` + +**Note:** editor exports have no top-level `type` field. The definition-type +discriminator (`"STL"`) only appears when the definition is nested inside a Connect +API payload (see below). + +### 2. Connect API create payload — single definition + +What developers POST to the `MktDataTransform` Connect API when authoring +programmatically. The outer object is a `DataTransformInputRepresentation`; the +editor-export shape is nested inside under `definition`: + +```jsonc +{ + "name": "MyTransform", // Required — API name of the transform + "label": "My Transform", // Required — user-visible label + "description": "…", // Optional + "type": "BATCH", // DataTransformType: "BATCH" | "STREAMING" + "dataSpaceName": "default", // Data space the transform lives in + "definition": { + "version": "66.0", + "type": "STL", // DataTransformDefinitionType: "STL" | "SQL" | "DCSQL" | … + "nodes": { /* … */ }, + "ui": { /* … */ } + } +} +``` + +Two distinct `type` fields live at different nesting levels: +- Outer `type` = `DataTransformType` (batch vs streaming). +- Inner `definition.type` = `DataTransformDefinitionType` (STL for batch BDTs; SQL / + DCSQL for raw-SQL transforms; hidden variants exist but are not surfaced to users). + +### 3. Connect API create payload — multi-definition variant + +For transforms with multiple definitions. Same outer fields; instead of a single +`definition` object there's a `definitions` array: + +```jsonc +{ + "name": "…", "label": "…", "type": "BATCH", "dataSpaceName": "default", + "definitions": [ + { "version": "66.0", "type": "STL", "nodes": { /* … */ }, "ui": { /* … */ } }, + /* … additional definitions … */ + ] +} +``` + +v1 of this skill reads the first definition and explains only that one; multi-definition +explanations are out of scope. + +### Each node (inner shape, identical in all three outer shapes) + +```jsonc +{ + "action": "", // One of the 25 documented action strings below + "sources": ["NODE_A", "NODE_B"], // Array of node names (empty for load/roots) + "parameters": { /* action-specific */ }, + "schema": { /* optional — slice/rename at node boundary */ } +} +``` + +## The 25 canonical action types + +| `action` string | Purpose (one line) | +|---|---| +| `load` | Read rows from a DMO or DLO | +| `filter` | Keep rows matching structured filter criteria | +| `sqlFilter` | Keep rows matching a raw SQL predicate | +| `join` | Join two upstream streams by key | +| `formula` | Add derived columns via per-row SQL formulas | +| `computeRelative` | Window function over partitioned / ordered rows | +| `aggregate` | Group-by aggregation | +| `extractGrains` | Fan out rows across time grains | +| `extractTable` | Extract a table from a flatten-JSON output | +| `schema` | Rename / drop / reshape columns | +| `outputD360` | Write rows to a target DMO or DLO | +| `appendV2` | Union rows from multiple upstream streams (v2 variant) | +| `flatten` | Flatten nested structure | +| `flattenJson` | Flatten a JSON field into rows/columns | +| `split` | Split rows into multiple downstream branches | +| `typeCast` | Cast field types | +| `update` | Update records | +| `bucket` | Assign bucket labels to field values | +| `formatDate` | Reformat date values | +| `extension` | Run a custom extension node | +| `extensionFunction` | Run a custom extension function | +| `cdpPredict` | Apply a prediction model | +| `jsonAggregate` | Aggregate JSON fields | +| `save` | Save an intermediate result | +| `recommendation` | Apply a recommendation model | + +**For full parameter shapes, enums, and examples: see `references/bdt-node-catalog.md`.** +**For SFSQL functions used in `formula` expressions: see `references/bdt-function-catalog.md`.** +**For window functions used in `computeRelative`: see `references/bdt-window-functions.md`.** + +## Terminology + +- **DMO** — Data Model Object. Canonical modeled entity. Suffix convention: `__dlm`. +- **DLO** — Data Lake Object. Raw-lake table. Suffix convention: `__dll`. +- **SFSQL** — Salesforce's internal SQL dialect used in formula expressions and SQL filters. +- **DCSQL** — a related SQL dialect used for Spark push-down execution (you may see it as + an `expressionType` value). +- **`ssot__` prefix** — Salesforce Standard Schema field, canonical / semantic. +- **`__c` suffix** — Custom field. +- **`KQ_` prefix** — Key-qualifier pattern (external ID / composite-key helper). Observed in + practice; not always documented. +- **UI label vs. JSON action name** — The JSON uses concise strings (e.g., `computeRelative`); + the BDT editor shows friendlier labels (e.g., "Window Transform"). The `ui.nodes[X].label` + field in the JSON carries the user-specified human-readable name per node. **Always use + the UI label in explanations when present.** + +## Key enums (for quick orientation) + +- **`DataTransformType`**: `BATCH`, `STREAMING` +- **`JoinType`**: `INNER`, `OUTER`, `LEFT_OUTER`, `RIGHT_OUTER`, `LOOKUP`, `MULTI_VALUE_LOOKUP`, `CROSS` +- **`WriteModeEnum`** (outputD360): `APPEND`, `MERGE`, `OVERWRITE`, `MERGE_UPSERT_DELETE`, `DELETE_ONLY` +- **`D360OutputTypeEnum`**: `dataLakeObject`, `dataModelObject` +- **`SliceMode`**: `SELECT`, `DROP` +- **`AggregateType`**: `UNIQUE`, `SUM`, `AVG`, `COUNT`, `MAX`, `MIN`, `MEDIAN`, `STDDEVP`, `STDDEV`, `VARP`, `VAR` +- **`DataType`**: `TEXT`, `NUMBER`, `DATE_ONLY`, `DATETIME`, `BOOLEAN` +- **`SortDirection`**: `ASC`, `DESC` + +For exhaustive enum lists with semantics: `bdt-node-catalog.md` or (for the raw extraction) +`research/bdt-schema-canonical.md`. + +## How this skill uses these references + +- `bdt-reference.md` (this file) — always in context. +- `bdt-node-catalog.md` — consulted before explaining any node type not previously covered + in the current conversation. +- `bdt-function-catalog.md` — consulted before interpreting any `formulaExpression` (job I1) + or reasoning about conditional logic (I2). +- `bdt-window-functions.md` — consulted whenever a `computeRelative` node is narrated. + +**If a node type or function isn't in any of the reference files**, the skill narrates what's +visible in the JSON and explicitly flags the item as undocumented in the materials available. +It does not guess at semantics. diff --git a/skills/explaining-batch-data-transform/references/bdt-window-functions.md b/skills/explaining-batch-data-transform/references/bdt-window-functions.md new file mode 100644 index 0000000..664a630 --- /dev/null +++ b/skills/explaining-batch-data-transform/references/bdt-window-functions.md @@ -0,0 +1,73 @@ +# Window Functions — for `computeRelative` nodes + +> **Last synced:** 2026-04-23 from the SFSQL window-functions reference and the BDT +> canonical schema. Consult this file whenever narrating a `computeRelative` node or +> explaining a window-function expression. + +## When this applies + +`computeRelative` nodes evaluate a **window function** over rows. The `parameters`: + +```jsonc +{ + "partitionBy": ["ssot__AccountId__c"], // → SQL `PARTITION BY` + "orderBy": [ // → SQL `ORDER BY` + {"fieldName": "ssot__CreatedDate__c", "direction": "ASC"} + ], + "expressionType": "SQL", // or "DCSQL" + "fields": [ + { + "name": "OrderRank__c", + "formulaExpression": "row_number()", // the window function call + "type": "NUMBER", "businessType": "Number", + "precision": 18, "scale": 0 + } + ] +} +``` + +The `formulaExpression` names the window function; partitioning and ordering come from the +top-level `partitionBy` and `orderBy`. **A computeRelative node may include at most one +compute-relative function per expression** (per upstream BDT docs). + +## Available window functions + +| Function | Returns | What it does | +|---|---|---| +| `row_number()` | NUMBER | 1, 2, 3… for each row in its partition, in the given order. Non-deterministic when sort keys tie. | +| `rank()` | NUMBER | Like `row_number` but peers share a rank; next rank after N peers is N+1 (gaps). | +| `dense_rank()` | NUMBER | Like `rank` but no gaps — consecutive integers even with ties. | +| `percent_rank()` | NUMBER | `(rank - 1) / (partition_rows - 1)` — relative rank within partition, 0 to 1. | +| `cume_dist()` | NUMBER | Cumulative distribution: fraction of partition rows at or before current. | +| `ntile(n)` | NUMBER | Bucket number 1..n, dividing partition rows as evenly as possible. | +| `lag(value)` / `lag(value, offset)` / `lag(value, offset, default)` | same as value | Value at offset rows *before* current (default offset=1; default if no such row is NULL unless a default is supplied). | +| `lead(value)` / `lead(value, offset)` / `lead(value, offset, default)` | same as value | Symmetric with `lag` but looks forward. | +| `first_value(value)` | same as value | Value at the first row of the current window frame. | +| `last_value(value)` | same as value | Value at the last row of the frame. Default frame ends at "current + peers", which is often *not* what users want. | +| `nth_value(value, n)` | same as value | Value at the nth row of the frame (counting from 1). NULL if no such row. | +| Any aggregate with `OVER(...)` | depends | Runs the aggregate over the window (running sum, etc.). | + +## How this maps to BDT JSON + +- `partitionBy` is the SQL `PARTITION BY` — the columns that group rows into windows. +- `orderBy` is the SQL `ORDER BY` — the ordering within each partition. +- **Peers** are rows with identical sort keys. +- Default **frame** (when not otherwise specified): rows from the first row of the partition + through the current row's last peer. For `last_value` and `nth_value` this is often not + the user's intent — narrate accordingly. + +## Common narration patterns + +- **`row_number()` partitioned by X** → "numbers each row within the same X, in the order + given by `orderBy`." +- **`rank() partitioned by X order by Y`** → "ranks rows within each X group by Y; ties + share a rank and the next rank has gaps." +- **`case when row_number()=1 then VAL else 0 end`** → "keeps VAL only on the first ranked + row per partition; everything else is 0. This is the canonical 'first-occurrence extract' + idiom." + +## Sources + +- SFSQL window-functions reference (internal Data Cloud / SDB SFSQL docs). +- BDT canonical schema: `ComputeRelativeParametersInputRepresentation`, + `ComputeRelativeSortParametersInputRepresentation`.