feat: launcher orchestration support (service-key auth, current-task, ParentRunId tagging)

src/cwl_platform/__init__.py

@@ -15,6 +15,26 @@
     'NGS360': NGS360Platform
 }
 
+# Substrings that mark an environment variable as carrying a credential. The
+# environment dump below goes to whatever collects the container's logs, which
+# is a wider audience than whoever provisioned the credential.
+_SECRET_NAME_MARKERS = ('TOKEN', 'KEY', 'SECRET', 'PASSWORD')
+
+
+def _redact(name, value):
+    '''
+    Return an environment variable's value, masked if the name marks it secret
+
+    :param name: Environment variable name
+    :param value: Environment variable value
+    :return: The value, or a mask of it
+    '''
+    if any(marker in name.upper() for marker in _SECRET_NAME_MARKERS):
+        # Length is kept because "set but empty" and "set to a truncated value"
+        # are both real misconfigurations worth telling apart in a log.
+        return f"<redacted, {len(value)} chars>"
+    return value
+
 
 class PlatformFactory():
     ''' PlatformFactory '''
@@ -36,7 +56,7 @@ def detect_platform(self):
         # print out environment variables and raise an error
         logging.info("Environment Variables:")
         for name, value in os.environ.items():
-            logging.info("%s: %s", name, value)
+            logging.info("%s: %s", name, _redact(name, value))
 
         raise ValueError("Unable to detect platform")
 

src/cwl_platform/ngs360_platform.py

@@ -3,6 +3,20 @@
 
 This module implements the Platform abstract base class using the GA4GH
 Workflow Execution Service (WES) API.
+
+Environment contract, set by whoever starts the launcher container:
+
+    WES_API_ENDPOINT    WES API base URL, including the /ga4gh/wes/v1 prefix
+    NGS360_API_ENDPOINT NGS360 API base URL (projects, file upload)
+    NGS360_AUTH_TOKEN   Bearer token for a user; preferred when available
+    WES_SERVICE_KEY     Shared service key, used only when no bearer token is
+                        set -- for headless runs with no user session to source
+                        a token from. See connect() for the trust implications.
+    WES_ON_BEHALF_OF    User the service key is acting for; audit trail only
+    WES_RUN_ID          This launcher's own WES run id, when the launcher is
+                        itself running as a WES run (e.g. as an AWS Batch job).
+                        Makes get_current_task() work and links every task this
+                        launcher submits back to it.
 """
 
 import os
@@ -86,6 +100,8 @@ def __init__(self, name):
 
         self.ngs360_endpoint = None
         self._ngs360_auth_config = {}
+        self._wes_service_key = None
+        self._wes_on_behalf_of = None
 
         self.projects = {}  # Map project names to project objects
         self.workflows = {}  # Map workflow names to workflow objects
@@ -98,6 +114,9 @@ def connect(self, **kwargs):
         :param kwargs: Connection parameters
             - api_endpoint: WES API endpoint URL
             - ngs360_auth_token: Authentication token for the WES API
+            - wes_service_key: Shared service key, used only when no
+              ngs360_auth_token is available
+            - wes_on_behalf_of: User the service key is acting for
         """
         self.connected = False
 
@@ -112,10 +131,28 @@ def connect(self, **kwargs):
             "ngs360_auth_token",
             os.environ.get("NGS360_AUTH_TOKEN")
         )
+        # The service key is a fallback for headless runs: a launcher running as an
+        # AWS Batch job has no user session to source a token from, so whoever
+        # submits the job has to inject a credential, and the service key is the
+        # one WES already provisions for callers acting without a user session.
+        # A bearer token wins when both are set: it names one real user and is
+        # scoped to them, whereas a holder of the service key is trusted to
+        # assert whatever identity it likes, so injecting the key into a
+        # container widens WES's trust boundary to that container.
+        service_key = kwargs.get(
+            "wes_service_key",
+            os.environ.get("WES_SERVICE_KEY")
+        )
         if auth_token:
             self._ngs360_auth_config['token'] = auth_token
+        elif service_key:
+            self._wes_service_key = service_key
+            self._wes_on_behalf_of = kwargs.get(
+                "wes_on_behalf_of",
+                os.environ.get("WES_ON_BEHALF_OF")
+            )
         else:
-            raise ValueError("NGS360 AUTH TOKEN is required")
+            raise ValueError("NGS360 AUTH TOKEN or WES SERVICE KEY is required")
 
         # Test connection by getting service info
         try:
@@ -137,6 +174,17 @@ def connect(self, **kwargs):
         if not self.ngs360_endpoint:
             raise ValueError("NGS360 API endpoint URL is required")
 
+        # /api/v1/auth/me validates a user's bearer token, so there is nothing to
+        # probe when authenticating with the service key. The service-info call
+        # above already proved we can reach and authenticate against WES.
+        if self._wes_service_key:
+            self.logger.info(
+                "Authenticated to WES with a service key on behalf of '%s'",
+                self._wes_on_behalf_of or "(unspecified)"
+            )
+            self.connected = True
+            return self.connected
+
         # Test connection by getting current user info
         try:
             response = requests.get(
@@ -177,6 +225,10 @@ def _make_request(self, method, path, **kwargs):
 
         if 'token' in self._ngs360_auth_config:
             headers["Authorization"] = f"Bearer {self._ngs360_auth_config['token']}"
+        elif self._wes_service_key:
+            headers["X-Internal-Service-Key"] = self._wes_service_key
+            if self._wes_on_behalf_of:
+                headers["X-On-Behalf-Of"] = self._wes_on_behalf_of
 
         response = requests.request(
             method=method,
@@ -399,9 +451,31 @@ def get_current_task(self):
         """
         Get the current task
 
-        Note: WES API doesn't have a concept of current task, so this returns None
-        """
-        return None
+        A launcher that is itself a WES run -- e.g. one running as an AWS Batch
+        job -- is told its own run id through WES_RUN_ID, the same way the
+        SevenBridges platform uses TASK_ID. Its own run record is where the
+        launcher reads back the inputs it was started with.
+
+        :return: WESTask object for this launcher's own run
+        :raises ValueError: If WES_RUN_ID is not set
+        """
+        run_id = os.environ.get("WES_RUN_ID")
+        if not run_id:
+            raise ValueError("ERROR: Environment variable WES_RUN_ID not set.")
+        self.logger.info("WES_RUN_ID: %s", run_id)
+
+        response = self._make_request("GET", f"runs/{run_id}")
+        request = response.get("request") or {}
+
+        return WESTask(
+            run_id=run_id,
+            name=response.get("name") or "",
+            state=self.STATE_MAP.get(response.get("state", "UNKNOWN"), "Unknown"),
+            # workflow_params are what the launcher was submitted with, which is
+            # what get_task_input reads back.
+            inputs=request.get("workflow_params") or {},
+            outputs=response.get("outputs") or {},
+        )
 
     def get_task_cost(self, task):
         """
@@ -639,16 +713,25 @@ def submit_task(self, name, project, workflow, parameters, execution_settings=No
         else:
             files = None
 
+        tags = {
+            "ProjectId": project["project_id"],
+            "TaskName": name,
+        }
+        # When this launcher is itself a WES run, link what it submits back to it.
+        # WES promotes ParentRunId into an indexed column, which is what lets it
+        # roll up launcher progress without the launcher reporting its own status,
+        # and what lets a restarted launcher find work it already submitted.
+        launcher_run_id = os.environ.get("WES_RUN_ID")
+        if launcher_run_id:
+            tags["ParentRunId"] = launcher_run_id
+
         # Prepare the request data
         data = {
             "workflow_params": json.dumps(parameters),
             "workflow_type": workflow_type,
             "workflow_type_version": workflow_type_version,
             "workflow_url": workflow_url,
-            "tags": json.dumps({
-                "ProjectId": project["project_id"],
-                "TaskName": name,
-            }),
+            "tags": json.dumps(tags),
             "workflow_engine_parameters": json.dumps(
                 workflow_engine_parameters
             ),