diff --git a/docs.json b/docs.json index 8e5023336..b0866d93c 100644 --- a/docs.json +++ b/docs.json @@ -6140,6 +6140,7 @@ ] }, "v2.1.x-SNAPSHOT/how-to-guides/admin-guide/cli-ingestion-with-basic-auth", + "v2.1.x-SNAPSHOT/how-to-guides/admin-guide/data-retention", "v2.1.x-SNAPSHOT/how-to-guides/admin-guide/reindexing-search", { "group": "Preferences", @@ -6468,6 +6469,7 @@ ] }, "v2.1.x-SNAPSHOT/how-to-guides/data-governance/intake-forms", + "v2.1.x-SNAPSHOT/how-to-guides/data-governance/governance-workflows", "v2.1.x-SNAPSHOT/how-to-guides/data-governance/metrics", "v2.1.x-SNAPSHOT/how-to-guides/data-governance/column-bulk-operations" ] diff --git a/public/images/how-to-guides/admin-guide/data-retention/automation-workflow-retention.png b/public/images/how-to-guides/admin-guide/data-retention/automation-workflow-retention.png new file mode 100644 index 000000000..36a04d713 Binary files /dev/null and b/public/images/how-to-guides/admin-guide/data-retention/automation-workflow-retention.png differ diff --git a/public/images/how-to-guides/admin-guide/data-retention/configuration-tab.png b/public/images/how-to-guides/admin-guide/data-retention/configuration-tab.png new file mode 100644 index 000000000..57c25a41b Binary files /dev/null and b/public/images/how-to-guides/admin-guide/data-retention/configuration-tab.png differ diff --git a/public/images/how-to-guides/governance/workflows/data-product-asset-count-condition.png b/public/images/how-to-guides/governance/workflows/data-product-asset-count-condition.png new file mode 100644 index 000000000..7c70b8455 Binary files /dev/null and b/public/images/how-to-guides/governance/workflows/data-product-asset-count-condition.png differ diff --git a/public/images/how-to-guides/governance/workflows/data-product-output-port-count-condition.png b/public/images/how-to-guides/governance/workflows/data-product-output-port-count-condition.png new file mode 100644 index 000000000..a834f721e Binary files /dev/null and b/public/images/how-to-guides/governance/workflows/data-product-output-port-count-condition.png differ diff --git a/public/images/how-to-guides/governance/workflows/workflow-column-trigger-config.png b/public/images/how-to-guides/governance/workflows/workflow-column-trigger-config.png new file mode 100644 index 000000000..b91fc23c2 Binary files /dev/null and b/public/images/how-to-guides/governance/workflows/workflow-column-trigger-config.png differ diff --git a/public/images/how-to-guides/governance/workflows/workflow-trigger-fields.png b/public/images/how-to-guides/governance/workflows/workflow-trigger-fields.png new file mode 100644 index 000000000..86deea8a8 Binary files /dev/null and b/public/images/how-to-guides/governance/workflows/workflow-trigger-fields.png differ diff --git a/public/images/how-to-guides/roles-policies/impersonated-by-policy-condition.png b/public/images/how-to-guides/roles-policies/impersonated-by-policy-condition.png new file mode 100644 index 000000000..2e8712899 Binary files /dev/null and b/public/images/how-to-guides/roles-policies/impersonated-by-policy-condition.png differ diff --git a/public/images/how-to-guides/roles-policies/organization-policy-rules.png b/public/images/how-to-guides/roles-policies/organization-policy-rules.png new file mode 100644 index 000000000..017b5e2b1 Binary files /dev/null and b/public/images/how-to-guides/roles-policies/organization-policy-rules.png differ diff --git a/public/images/how-to-guides/roles-policies/rbac-search-setting.png b/public/images/how-to-guides/roles-policies/rbac-search-setting.png new file mode 100644 index 000000000..5b9f015e3 Binary files /dev/null and b/public/images/how-to-guides/roles-policies/rbac-search-setting.png differ diff --git a/public/images/how-to-guides/roles-policies/roles-list.png b/public/images/how-to-guides/roles-policies/roles-list.png new file mode 100644 index 000000000..c4198f41f Binary files /dev/null and b/public/images/how-to-guides/roles-policies/roles-list.png differ diff --git a/public/images/how-to-guides/roles-policies/service-attributes-edit.png b/public/images/how-to-guides/roles-policies/service-attributes-edit.png new file mode 100644 index 000000000..0f68d2ca1 Binary files /dev/null and b/public/images/how-to-guides/roles-policies/service-attributes-edit.png differ diff --git a/public/images/how-to-guides/roles-policies/service-attributes.png b/public/images/how-to-guides/roles-policies/service-attributes.png new file mode 100644 index 000000000..d0b09901b Binary files /dev/null and b/public/images/how-to-guides/roles-policies/service-attributes.png differ diff --git a/public/images/how-to-guides/roles-policies/service-connection-tab.png b/public/images/how-to-guides/roles-policies/service-connection-tab.png new file mode 100644 index 000000000..93aabb02d Binary files /dev/null and b/public/images/how-to-guides/roles-policies/service-connection-tab.png differ diff --git a/public/images/how-to-guides/roles-policies/service-environment-policy-condition.png b/public/images/how-to-guides/roles-policies/service-environment-policy-condition.png new file mode 100644 index 000000000..0c7f3c927 Binary files /dev/null and b/public/images/how-to-guides/roles-policies/service-environment-policy-condition.png differ diff --git a/public/images/how-to-guides/roles-policies/view-all-rule.png b/public/images/how-to-guides/roles-policies/view-all-rule.png new file mode 100644 index 000000000..b2935320b Binary files /dev/null and b/public/images/how-to-guides/roles-policies/view-all-rule.png differ diff --git a/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/data-retention.mdx b/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/data-retention.mdx new file mode 100644 index 000000000..9d416ef3d --- /dev/null +++ b/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/data-retention.mdx @@ -0,0 +1,51 @@ +--- +title: Configure Data Retention +description: Configure how long OpenMetadata keeps automation workflows and other operational records. +sidebarTitle: Data Retention +--- + +# Configure Data Retention + +The Data Retention application removes operational records after their configured retention periods. In OpenMetadata 2.1, it also cleans up short-lived Automation Workflow records. Test Connection, Query Runner, and Reverse Ingestion runs create these records. + +## Configure Automation Workflow Retention + +You need administrator access to configure and run the Data Retention application. + +1. Go to **Settings** > **Applications** > **Data Retention**. +2. Click **Configuration**. + + Data Retention application with the Configuration tab highlighted + +3. Scroll to **Automation Workflow Retention Period (days)**. +4. Set the retention period: + - Enter a positive integer to delete Automation Workflow records older than that many days. + - Enter `0` to retain Automation Workflow records indefinitely. + + Automation Workflow Retention Period configured for 30 days +5. Save the configuration. +6. Return to the Data Retention application details. +7. Optional: To apply the policy without waiting for the next scheduled run, select **Run Now**. + + +The default retention period is `30` days. Cleanup uses each Workflow record's last-updated time, so a record that has remained in `Pending` or `Running` beyond the retention period is also eligible for deletion. + + +Deleting expired Automation Workflow records is irreversible. This setting defaults to `30` days even for an application configuration saved before the field existed. After an upgrade, the next scheduled Data Retention run can therefore remove existing records older than 30 days. Set the value to `0` before that run if you must retain them. + + +Removing these records does not delete the service that a workflow used. OpenMetadata also cleans up the Workflow's related ownership records and stored connection secrets. + + +One application run deletes at most 100,000 Automation Workflow records. Run the application again when an older installation has a larger backlog. + + +## Related Retention Settings + +The same application configures retention for other record types. It covers change events and activity records. It also covers test results, profile data, and audit logs. Each field is measured in days. Review every value before you run the application in a production environment. diff --git a/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.mdx b/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.mdx index 6c8b40d0a..60afab04c 100644 --- a/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.mdx +++ b/v2.1.x-SNAPSHOT/how-to-guides/admin-guide/roles-policies/authorization.mdx @@ -11,12 +11,13 @@ OpenMetadata controls access through three building blocks: rules define what is ## Rules A rule in a policy is the building block of authorization. It contains the following: -1. **Name**: A unique name to define the rule -2. **Description**: Description of the rule -3. **Resources**: List of resources this rule applies to. An admin can select a specific resource, such as Table or All, to apply against all resources. -4. **Operations**: List of operations this rule applies to. An admin can select a specific operation such as EditOwners or All to apply against all the operations. -5. **Condition**: Expressions written using policy functions that evaluate true or false. -6. **Effect**: Deny or Allow the operation. + +- **Name**: A unique name for the rule. +- **Description**: A description of the rule. +- **Resources**: The resources to which the rule applies. Select a specific resource, such as `Table`, or select `All` to apply the rule to every resource. +- **Operations**: The operations to which the rule applies. Select a specific operation, such as `EditOwners`, or select `All` to apply the rule to every operation. +- **Effect**: Determines whether the rule allows or denies the selected operations. +- **Condition**: An expression that uses policy functions and evaluates to true or false. Building Blocks of Authorization: Rules @@ -69,11 +70,171 @@ noOwner() && matchAllTags('PersonalData.Personal', 'Tier.Tier1') This condition is true only if the asset has no owner and carries both of the listed tags. +### Use Service Attributes in Policy Conditions + +Every service can store three deployment attributes. + +| Attribute | Supported value | Policy support | +|---|---|---| +| **Environment** | `Development`, `Staging`, `QA`, `UAT`, `Production`, `Sandbox`, or `Other` | Use `matchAnyServiceEnvironment(...)`. | +| **Region** | Free text, such as `us-east-1` | Display only. It is not available as a policy condition. | +| **Deployment** | Free text, such as `prod-cluster-01` | Display only. It is not available as a policy condition. | + +#### Set Service Attributes + +You need permission to edit the service. + +1. Go to **Settings** > **Services**. +2. Select the service category, then open the service you want to configure. +3. Click the **Connection** tab. + + Connection tab on a service page + +4. In the **Service Attributes** section, click **Edit**. + + Service Attributes section with the Edit button highlighted + +5. Complete the following steps: + - Select an **Environment**. + - Optional: Enter the **Region** and **Deployment** identifier. + + Service Attributes configured with an environment, region, and deployment + +6. Click **Save**. + +The attributes apply to the service. You do not need to add them separately to every asset ingested from that service. + +Service-aware conditions let one rule cover the service and the assets it ingests: + +| Condition | Returns true when | +|---|---| +| `matchAnyServiceTag('Environment.Development')` | The service has any listed tag or glossary term. | +| `matchAnyServiceType('Snowflake', 'BigQuery')` | The service has any listed connector type. Matching is case-insensitive. | +| `matchAnyServiceName('snowflake-sandbox')` | The asset belongs to any listed service name. Use the service name, not its fully qualified name. | +| `matchAnyServiceEnvironment('Development', 'Sandbox')` | The service has any listed environment. Matching is case-insensitive. | + +The referenced value must exist when you save the policy. These conditions return false for resources that do not belong to a service. Examples include glossary terms and users. + +For example, the following condition can be used in a `Deny` rule to prevent access to assets from development services: + +``` +matchAnyServiceEnvironment('Development') +``` + +Apply the rule to the required resources and operations. A service is treated as its own service, so a matching `Deny` rule can hide both the service and its assets. + +#### Create a Service-Scoped Policy Rule + +Test the rule in an unassigned policy before applying it to users or teams. + +1. Go to **Settings** > **Access Control** > **Policies**. +2. Create a policy or open an existing policy. +3. Click **Add Rule**. +4. Complete the following steps: + - Enter the **Rule Name**. + - Optional: Enter a **Description**. + - Select the required **Resources** and **Operations**. For example, select **All** and `ViewAll` to evaluate view access across service-backed resources. + - Set **Effect** to `Deny` or `Allow`. When rules conflict, `Deny` takes precedence. + - In **Condition**, enter a service-aware expression. For example: + + ```text + matchAnyServiceEnvironment('Development', 'Sandbox') + ``` + Confirm that the editor reports **Valid condition**. + + Policy rule with a valid service environment condition +5. Select **Create**. + +Review the policy with the Permission Debugger before assigning it to a production team. + + +Service-aware conditions apply to search-backed results, direct entity requests, and create operations. A plain entity list API, such as `GET /v1/tables`, does not resolve an individual entity for its authorization check and cannot evaluate these conditions per result. Do not use an unfiltered list endpoint as a security boundary for a service-scoped policy. + + +### Understand Inherited Service Tags + +Tags now pass down automatically through the service hierarchy. For example, a tag on a database service is inherited by its databases, database schemas, and tables. + +- OpenMetadata marks inherited tags as `Derived`. +- Inherited tags are merged with the tags assigned directly to the child asset. +- If the child asset already has the same tag assigned directly, the child's tag takes precedence. + +You cannot remove a derived tag from a child asset. To remove it, go to the parent where the tag was assigned and remove it there. + +Policy conditions such as `matchAnyTag(...)` also evaluate inherited tags. + + +After you upgrade, reindex OpenMetadata so existing assets receive their inherited tags in search. Until reindexing finishes, an asset's own page can show inherited tags before they appear in Explore. + + +### Restrict Requests Made Through Bots + +Use impersonation conditions in `Deny` rules to restrict actions attributed to a bot acting on a person's behalf. This includes requests that use the `X-Impersonate-User` header and Model Context Protocol (MCP) or artificial intelligence (AI) tool calls. + +| Condition | Returns true when | +|---|---| +| `isImpersonated()` | The request is attributed to any bot. | +| `impersonatedBy('McpApplicationBot')` | The request is attributed to any named bot. Bot-name matching is case-insensitive. | + +Use this condition to match every request attributed to a bot: + +``` +isImpersonated() +``` + +To restrict a specific bot, use its existing bot name: + +``` +impersonatedBy('McpApplicationBot') +``` + +You can combine `isImpersonated()` with `matchAnyTag(...)` to protect tagged assets. The tag's fully qualified name must already exist in OpenMetadata or the condition will be invalid. Put `isImpersonated()` before the tag condition so ordinary requests can stop evaluation before OpenMetadata loads the tags. When you use `impersonatedBy(...)`, the supplied name must resolve to a bot or to its bot user. + +#### Create a Bot Attribution Rule + +Create the rule in an unassigned test policy first. A broad impersonation `Deny` rule can block MCP and AI tool access to matching resources. + +1. Go to **Settings** > **Access Control** > **Policies**. +2. Create a policy or open an existing policy. +3. Click **Add Rule**. +4. Complete the following steps: + - Enter the **Rule Name**. + - Optional: Enter a **Description**. + - Select the required **Resources** and **Operations**. + - Set **Effect** to `Deny`. + - In **Condition**, enter `isImpersonated()` or `impersonatedBy('')`. Confirm that the editor reports **Valid condition**. + + Deny rule with a valid impersonatedBy condition for McpApplicationBot + +5. Select **Create**. + + +Use the Permission Debugger to verify the expected result before assigning the policy. + + +These functions report bot attribution. They do not verify that an impersonation grant was authorized. Use them in `Deny` rules. They apply when OpenMetadata performs a per-entity authorization check, but they do not filter results returned directly by `search_metadata`, `semantic_search`, or `nlq_search`. + + ### Default Rules -Go to **Settings** > **Policies** > **Organization Policy** to see the three rules that come built in and apply to everyone in your organization. +Go to **Settings** > **Access Control** > **Policies** and select **Organization Policy** to see the three rules that come built in and apply to everyone in your organization. -Organization Policy showing the default rules +Organization Policy page showing its default rules - **OrganizationPolicy-NoOwner-Rule** @@ -93,6 +254,8 @@ Go to **Settings** > **Policies** > **Organization Policy** to see the three rul - **OrganizationPolicy-ViewAll-Rule** + Organization Policy - View All - Rule + This rule lets every user view every asset. It has no condition, so it applies to all users and all assets, whether or not they own the asset or belong to its team. This is why everyone can find and open any asset in Explore by default. With RBAC Search turned on, this rule still applies. See [Roles Based Access Controls (RBAC) Search](#roles-based-access-controls-rbac-search). @@ -145,7 +308,7 @@ For teams with compliance or regulatory requirements, it's good practice to make A role is a ready-made bundle of policies that matches a job function. Instead of assigning individual policies to every user, you assign them a role and they get all the right policies in one step. -Building Blocks of Authorization: Roles +Building Blocks of Authorization: Roles Roles can be assigned to individual users or to entire teams. When a role is assigned to a team, every member of that team gets it automatically – which makes onboarding and offboarding much simpler. @@ -181,7 +344,7 @@ By default, every user can search for and find any data asset in OpenMetadata, r To enable or disable this, go to **Settings** > **Preferences** > **Search**. -Roles Based Access Controls (RBAC) Search Option +Roles Based Access Controls (RBAC) Search Option ### What Changes When You Turn It On diff --git a/v2.1.x-SNAPSHOT/how-to-guides/data-governance.mdx b/v2.1.x-SNAPSHOT/how-to-guides/data-governance.mdx index 71a06882f..3ff7e62a6 100644 --- a/v2.1.x-SNAPSHOT/how-to-guides/data-governance.mdx +++ b/v2.1.x-SNAPSHOT/how-to-guides/data-governance.mdx @@ -20,4 +20,7 @@ Apart from well-defined access control roles and policies, a common vocabulary w Discover the essentials of domains and data products management. + + Trigger workflows on metadata changes and route Data Products by their contents. + diff --git a/v2.1.x-SNAPSHOT/how-to-guides/data-governance/governance-workflows.mdx b/v2.1.x-SNAPSHOT/how-to-guides/data-governance/governance-workflows.mdx new file mode 100644 index 000000000..592abd423 --- /dev/null +++ b/v2.1.x-SNAPSHOT/how-to-guides/data-governance/governance-workflows.mdx @@ -0,0 +1,94 @@ +--- +title: Configure Governance Workflow Triggers and Conditions +description: Trigger governance workflows for relevant entity changes and route Data Products by their asset and output port counts. +sidebarTitle: Governance Workflows +--- + +# Configure Governance Workflow Triggers and Conditions + +Governance workflows can respond to changes on specific data assets and route an entity based on its metadata. OpenMetadata 2.1 adds entity-specific trigger fields, support for column custom-property changes, and Data Product count conditions. + +You need permission to create or edit a Workflow Definition. You also need the `Deploy` operation to deploy the completed workflow. + +## Choose Which Changes Trigger a Workflow + +For an event-based start node, the **Include Fields** and **Exclude Fields** lists contain only fields that apply to the selected data asset types. The lists combine: + +- Common fields, such as `description`, `owners`, and `tags`. +- Fields for the selected entity type, such as `columns` for a Table or `outputPorts` for a Data Product. +- Custom properties defined for the selected entity type. The selector marks these entries as **Custom Property**. + +This keeps a workflow from offering fields that cannot change on the selected entity type. + +Include Fields list showing the columns field for a Table workflow trigger + + +If **Include Fields** is empty, any supported common or entity-specific field can trigger the workflow. Use **Include Fields** when the workflow should respond only to a smaller set of changes. + + +### Trigger a Workflow When a Column Custom Property Changes + +Use the Table `columns` field to respond when a custom property on any column changes. + +1. Go to **Govern** > **Workflows**, create or edit a workflow, and select the start node. +2. Under **Data Asset**, select **Table**. Select **Dashboard Data Model** instead when the custom property belongs to a dashboard data model column. +3. Set **Trigger Type** to **Event Based**. +4. Under **Include Fields**, select `columns`. +5. Complete the workflow, save it, and deploy it. + +Start node configured to monitor the columns field on Tables + +Changing a column custom property now creates a column change on the parent entity and starts the workflow. One update to a table starts the workflow at most once, even when the update changes multiple columns. + + +The trigger watches the `columns` field as a whole. It cannot be limited to one column or one column custom property. The condition builder does not offer `columns` because it is an array of column objects and cannot be compared as a single value. + + +## Route Data Products by Asset or Output Port Count + +The **Check Condition** node provides two numeric fields when the workflow evaluates a Data Product: + +| Field | Internal name | What it counts | +|---|---|---| +| **Asset Count** | `assetsCount` | Assets assigned to the Data Product | +| **Output Port Count** | `outputPortsCount` | Assets configured as output ports | + +For example, use `Asset Count > 0` to continue only when the Data Product contains an asset, or `Output Port Count > 0` to require at least one output port. + +1. Configure the start node with **Data Product** selected under **Data Asset**. The count fields are not available when the start node processes Tables or another entity type. +2. Add a **Check Condition** node. If you changed the start node after adding this node, close and reopen its configuration so the field list refreshes. +3. In the condition builder's **Field** list, select **Asset Count** or **Output Port Count**. Do not select **Data Product**. That option represents an entity relationship and provides operators such as **Is** and **Is Not**. +4. Select `>` and enter `0` to require at least one item. +5. Connect the true and false branches to the appropriate next steps. +6. Save and deploy the workflow. + +Check Condition configured with Asset Count greater than zero + +Check Condition configured with Output Port Count greater than zero + +OpenMetadata calculates these counts when the condition runs. They are available for workflow evaluation but are not stored as fields on the Data Product. + + +To validate both requirements, add conditions for `assetsCount > 0` and `outputPortsCount > 0`, then use the branch that matches your approval process. + + +## Verify the Workflow + +Test the deployed workflow with a non-production entity: + +1. Change a selected trigger field on the entity. +2. Confirm that a workflow instance starts. +3. For a Data Product condition, test once with an empty collection and once after adding an asset or output port. +4. Confirm that each instance follows the expected true or false branch. diff --git a/v2.1.x-SNAPSHOT/how-to-guides/data-marketplace/data-products.mdx b/v2.1.x-SNAPSHOT/how-to-guides/data-marketplace/data-products.mdx index e7c736aef..480408aa8 100644 --- a/v2.1.x-SNAPSHOT/how-to-guides/data-marketplace/data-products.mdx +++ b/v2.1.x-SNAPSHOT/how-to-guides/data-marketplace/data-products.mdx @@ -119,6 +119,8 @@ Changes to the following Data Product fields trigger configured governance appro - **Glossary Terms**: Assigning or removing glossary terms. If you have governance workflows configured to watch Data Product changes, they trigger on port and glossary term updates. + +You can also use **Asset Count** and **Output Port Count** in a workflow's **Check Condition** node. For configuration steps, see [Configure Governance Workflow Triggers and Conditions](/v2.1.x-SNAPSHOT/how-to-guides/data-governance/governance-workflows). ## View Data Assets in the Data Product from the Explore Page