Add get_template_attributes endpoint

custom-recipes/pi-system-af-tree/recipe.py

@@ -1,11 +1,11 @@
 import dataiku
-import json
 from dataiku.customrecipe import get_recipe_config, get_output_names_for_role
 from safe_logger import SafeLogger
 from osisoft_plugin_common import (
     get_credentials, PerformanceTimer
 )
 from osisoft_constants import OSIsoftConstants
+from osisoft_client import OSIsoftClient
 
 
 logger = SafeLogger("pi-system plugin", forbiden_keys=["token", "password"])
@@ -69,6 +69,7 @@ def filter_dictionary_keys(input_dict, keys_to_filter):
     {'name': 'paths', 'type': 'string'},
     {'name': 'id', 'type': 'string'},
     {'name': 'url', 'type': 'string'},
+    {'name': 'endpoint_url', 'type': 'string'},
     {'name': 'data_type', 'type': 'string'},
     {'name': 'summary_type', 'type': 'string'},
     {'name': 'boundary_type', 'type': 'string'},
@@ -82,13 +83,47 @@ def filter_dictionary_keys(input_dict, keys_to_filter):
 output_dataset.write_schema(schema)
 
 selectedAttributes = config.get("outputSelectedAttributes", [])
+
+client = OSIsoftClient(
+    server_url, auth_type, username, password,
+    is_ssl_check_disabled=is_ssl_check_disabled,
+    network_timer=network_timer
+)
+
 with output_dataset.get_writer() as writer:
-    for item in selectedAttributes:
-        if item.get("checked", True) is True:
-            item["category_names"] = json.dumps(item.get("category_names", []))
-            item["summary_type"] = json.dumps(item.get("summary_type", []))
-            item["paths"] = json.dumps(item.get("paths", []))
-            writer.write_row_dict(item)
+    while selectedAttributes:
+        selectedAttribute = selectedAttributes.pop()
+        selectedAttribute["url"] = None
+        selected_attribute_url = selectedAttribute.get("url")
+        if selected_attribute_url:
+            kwarg = {
+                "url": "{}?associations=Paths".format(selected_attribute_url),
+            }
+        else:
+            attribute_path = selectedAttribute.get("path")
+            search_url = client.endpoint.get_base_url() + "/attributes?associations=Paths&path={}".format(attribute_path)
+            # https://server/piwebapi/attributes?path=\\server\factory\INST-001-Temperature|InstrumentType
+            kwarg = {
+                "url": search_url,
+            }
+        # last one is never pushed
+        response = client.push_to_batch(selectedAttribute, **kwarg)
+        if not selectedAttributes:
+            # Reached the last element, forcing the flush
+            for selectedAttribute_back, reply in response:
+                data_type = selectedAttribute_back.get("data_type")
+                selectedAttribute_back["endpoint_url"] = reply.get("Content", {}).get("Links", {}).get(data_type)
+                selectedAttribute_back["paths"] = reply.get("Paths")
+                selectedAttribute_back["id"] = reply.get("WebId")
+                writer.write_row_dict(selectedAttribute_back)
+            response = client.flush_batch()
+
+        for selectedAttribute_back, reply in response:
+            data_type = selectedAttribute_back.get("data_type")
+            selectedAttribute_back["endpoint_url"] = reply.get("Content", {}).get("Links", {}).get(data_type)
+            selectedAttribute_back["paths"] = reply.get("Content", {}).get("Paths")
+            selectedAttribute_back["id"] = reply.get("Content", {}).get("WebId")
+            writer.write_row_dict(selectedAttribute_back)
 
 processing_timer.stop()
 logger.info("Overall timer:{}".format(processing_timer.get_report()))

js/pi-system_treecontroller.js

@@ -133,7 +133,8 @@ class Cache {
                 this.db = request.result;
                 this.stores.forEach(storeName => {
                     if (!this.db.objectStoreNames.contains(storeName)) {
-                        this.db.createObjectStore(storeName, { keyPath: "id" });
+                        const keyPath = storeName === this.attributesStoreName ? "path" : "id";
+                        this.db.createObjectStore(storeName, { keyPath: keyPath });
                     }
                 })
             };
@@ -149,8 +150,8 @@ class Cache {
         });
     }
 
-    async getAttribute(attrId) {
-        return this.getObject(this.attributesStoreName, attrId);
+    async getAttributeByPath(attributePath) {
+        return this.getObject(this.attributesStoreName, attributePath);
     }
 
     async getElementTree() {
@@ -699,7 +700,7 @@ app.controller('AfExplorerFormCtrl', [
 
         $scope.getChildrenFromDB = function(item) {
             if (item.type === "template") {
-                return getAttributesForTemplate(item);
+                return populateTreeTemplateAttributes(item);
             }
             return $scope.callPythonDo({ method: "get_children_from_db", parent: item })
                 .then(function(data) {
@@ -719,7 +720,7 @@ app.controller('AfExplorerFormCtrl', [
                         attributeLoadPromises.push(
                             addAttributeToLoadedAttributes(attribute)
                         );
-                        item.attribute_children.push(attribute.id);
+                        item.attribute_children.push(attribute.path);
                     });
                     item.children = data.choices.filter(node => node.type === item.type);
                     item.children.forEach(child => {
@@ -731,7 +732,7 @@ app.controller('AfExplorerFormCtrl', [
                         return {
                             updatedNode: item,
                             loadedAttributes: loadedAttributes
-                        }
+                        };
                     });
                 });
         }
@@ -819,34 +820,61 @@ app.controller('AfExplorerFormCtrl', [
             return attributePath.split('|')?.[0];
         }
 
-        function getAttributesForTemplate(node) {
-            return $scope.callPythonDo({ method: "get_attribute_for_template", template_name: node.title}).then(
+        function getTemplateGenericAttributes(templateName) {
+            return $scope.callPythonDo({ method: "get_template_attributes", template_name: templateName}).then(
                 function(data) {
-                    console.log("get_attribute_for_template", data);
-                    node.attribute_children = [];
-                    const loadedAttributes = data.attributes.map(attribute => {
-                        const elementPath = getElementPathFromAttributePath(attribute.path);
-                        return {
-                            ...attribute,
-                            expanded: false,
-                            parent_element: getElementNameFromPath(elementPath),
-                            parent_element_path: elementPath
-                        };
-                    })
-                    loadedAttributes.forEach(attribute => {
-                            addAttributeToLoadedAttributes(attribute);
-                            node.attribute_children.push(attribute.id);
-                        }
-                    );
-                    cacheTemplateTree();
-                    return {
-                        updatedNode: node,
-                        loadedAttributes: loadedAttributes
-                    };
+                    return data.attributes;
                 }
             );
         }
 
+        function buildAttributePath(elementPath, attributeName) {
+            return elementPath + '|' + attributeName;
+        }
+
+        function getAttributesForTemplate(templateName) {
+            return Promise.all([
+                getTemplateGenericAttributes(templateName),
+                $scope.getElementsForTemplate(templateName)
+            ])
+            .then(([ genericAttributes, templateElements ]) =>
+                {
+                    return genericAttributes.map((genericAttribute) => {
+                        return templateElements.map((element) => {
+                            console.log(genericAttribute, element);
+                            return{
+                                path: buildAttributePath(element.path, genericAttribute.title),
+                                category_names: genericAttribute.category_names,
+                                value_type: genericAttribute.value_type,
+                                title: genericAttribute.title,
+                                description: genericAttribute.description,
+                                expanded: false,
+                                parent_element: element.title,
+                                parent_element_path: element.path,
+                                template_name: templateName
+                            };
+                        })
+                    }).flat();
+                }
+            )
+        }
+
+        function populateTreeTemplateAttributes(node) {
+            const templateName = node.title;
+            return getAttributesForTemplate(templateName).then((loadedAttributes) => {
+                node.attribute_children = [];
+                loadedAttributes.forEach((attribute) => {
+                    addAttributeToLoadedAttributes(attribute);
+                    node.attribute_children.push(attribute.path);
+                })
+                cacheTemplateTree();
+                return {
+                    updatedNode: node,
+                    loadedAttributes: loadedAttributes
+                };
+            })
+        }
+
         $scope.isTemplateAssociatedElementSelected = function(element) {
             return $scope.ui.clickedNodes.includes(element.url);
         }
@@ -857,6 +885,7 @@ app.controller('AfExplorerFormCtrl', [
                     console.log("get_elements_for_template", data);
                     $scope.elementsByTemplate[templateName] = data.elements;
                     cacheElementsByTemplate();
+                    return data.elements;
                 }
             );
         }
@@ -1068,7 +1097,7 @@ app.controller('AfExplorerFormCtrl', [
             // TODO: replace by weak link single loading logic
             return getChildrenIfMissing(node).then(node => {
                 $scope.attributeList = $scope.attributeList.filter(
-                    attribute => !node.attribute_children.includes(attribute.id)
+                    attribute => !node.attribute_children.includes(attribute.path)
                 );
             });
         }
@@ -1083,7 +1112,7 @@ app.controller('AfExplorerFormCtrl', [
                 if (!attribute?.parent_template_name && parentTemplateName) {
                     attribute.parent_template_name = parentTemplateName;
                 }
-                const isAlreadyPresent = $scope.attributeList.find(attr => attr.id === attribute.id);
+                const isAlreadyPresent = $scope.attributeList.find(attr => attr.path === attribute.path);
                 if (!isAlreadyPresent) {
                     enrichAttribute(attribute, node);
                     $scope.attributeList.push(attribute);
@@ -1112,10 +1141,10 @@ app.controller('AfExplorerFormCtrl', [
                 return loadAndAddChildrenAttributes(node);
             }
             return Promise.all(
-                node.attribute_children.map(attributeId => {
-                    return $scope.cache.getAttribute(attributeId).then(loadedAttribute => {
+                node.attribute_children.map(attributePath => {
+                    return $scope.cache.getAttributeByPath(attributePath).then(loadedAttribute => {
                         if (!loadedAttribute) {
-                            throw new Error("Could not load attribute " + attributeId + " from the cache");
+                            throw new Error("Could not load attribute " + attributePath + " from the cache by path");
                         }
                         return loadedAttribute;
                     });

python-lib/osisoft_client.py

@@ -42,6 +42,8 @@ def __init__(self, server_url, auth_type, username, password, is_ssl_check_disab
         self.is_debug_mode = is_debug_mode
         self.debug_level = None
         self.network_timer = network_timer
+        self.batch_requests_parameters = []
+        self.batch_post_processing = []
 
     def get_auth(self, auth_type, username, password):
         if auth_type == "basic":
@@ -816,6 +818,27 @@ def search_elements(self, database, name=None, description=None, category=None,
             params["startIndex"] = start_index
             json_response = self.get(url=url, headers=headers, params=params)
 
+    def get_template_attributes(self, database, template_name):
+        attributes = []
+        url = "{}/elementtemplates".format(database)
+        headers = self.get_requests_headers()
+        while url:
+            json_response = self.get(url=url, headers=headers, params={})
+            url = json_response.get("Links", {}).get("Next", None)
+            element_templates = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
+            for element_template in element_templates:
+                name = element_template.get("Name")
+                if name == template_name:
+                    next_url = element_template.get("Links", {}).get("AttributeTemplates")
+                    while next_url:
+                        json_response = self.get(url=next_url, headers=headers, params={})
+                        next_url = json_response.get("Links", {}).get("Next", None)
+                        attribute_templates = json_response.get(OSIsoftConstants.API_ITEM_KEY, [])
+                        for attribute_template in attribute_templates:
+                            attributes.append(get_item_details(attribute_template))
+                    return attributes
+        return attributes
+
     def batched_search(self, database, element_name, attribute_name, element_category,
                        attribute_category, template, restrict_to_elements,
                        elements_max_count=None, attributes_max_count=None):
@@ -898,6 +921,23 @@ def batched_search(self, database, element_name, attribute_name, element_categor
                 for sub_item in sub_items:
                     yield sub_item
 
+    def push_to_batch(self, post_processing, **requests_kwargs):
+        self.batch_requests_parameters.append(requests_kwargs)
+        self.batch_post_processing.append(post_processing)
+        if len(self.batch_requests_parameters) > 50:
+            response = self.flush_batch()
+            for row in response:
+                yield row
+        else:
+            return
+
+    def flush_batch(self):
+        response = self._batch_requests(self.batch_requests_parameters)
+        for post_processing, row in zip(self.batch_post_processing, response):
+            yield (post_processing, row)
+        self.batch_requests_parameters = []
+        self.batch_post_processing = []
+
     def build_element_query(self, **kwargs):
         element_query_keys = {
             "element_name": "Name:'{}'",

resource/browse_af_tree.py

@@ -60,6 +60,8 @@ def do(payload, config, plugin_config, inputs):
         return get_elements_for_template(client, payload, config)
     if method == "get_attribute_for_template":
         return get_attribute_for_template(client, payload, config)
+    if method == "get_template_attributes":
+        return get_template_attributes(client, payload, config)
     if method == "do_search":
         return do_search(client, payload, config, network_timer)
 
@@ -248,6 +250,22 @@ def get_attribute_for_template(client, payload, config):
     )
     return result
 
+def get_template_attributes(client, payload, config):
+    database_name = config.get("database_name")
+    template_name = payload.get("template_name", None)
+    logger.info(
+        "Start call [get_template_attributes] database_name={}, template_name={}".format(
+            database_name, template_name
+        )
+    )
+    template_attributes = client.get_template_attributes(database_name, template_name)
+    result = {"choices": [], "attributes": template_attributes}
+    logger.info(
+        "End call [get_template_attributes] database_name={}, template_name={}".format(
+            database_name, template_name
+        )
+    )
+    return result
 
 def do_search(client, payload, config, network_timer):
     logger.info(