Data Mesh Guide

Example: Data Product Manifest (YAML)

# orders-data-product/manifest.yaml
name: orders
domain: commerce
owner: [email protected]
version: 2.1.0

description: |
  Order transactions from all sales channels.
  Includes order items, totals, and fulfillment status.

slo:
  freshness: 1h           # Data no older than 1 hour
  availability: 99.9%     # Uptime SLA
  quality_score: 95%      # % of quality checks passing

schema:
  type: delta
  location: s3://data-products/commerce/orders/
  
access:
  - sql: "SELECT * FROM commerce.orders"
  - api: "https://data.company.com/commerce/orders"
  
lineage:
  sources:
    - system: shopify
      table: orders
    - system: pos
      table: transactions
      
data_contract:
  primary_key: order_id
  not_null: [order_id, customer_id, order_date, total]
  
quality_checks:
  - unique(order_id)
  - not_null(customer_id)
  - total >= 0
  - order_date <= current_date()

Example: Policy as Code (OPA/Rego)

# governance/policies/data_product.rego
package dataproduct

# All data products must have an owner
deny[msg] {
    not input.manifest.owner
    msg := "Data product must have an owner defined"
}

# PII columns must be tagged
deny[msg] {
    column := input.schema.columns[_]
    column.pii == true
    not column.tags["pii"]
    msg := sprintf("PII column %v must be tagged", [column.name])
}

# SLO freshness must be defined
deny[msg] {
    not input.manifest.slo.freshness
    msg := "Data product must define freshness SLO"
}

# Minimum documentation required
deny[msg] {
    count(input.manifest.description) < 50
    msg := "Data product description must be at least 50 characters"
}