feat: add query() time_unit option, list_streams(), list_alerts(), docs fix

docs/notebooks/Openobserve-DataConnector.ipynb

@@ -75,27 +75,6 @@
     "```"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream"
-    }
-   ],
-   "source": [
-    "# Check we are running Python 3.6\n",
-    "import sys\n",
-    "\n",
-    "MIN_REQ_PYTHON = (3, 6)\n",
-    "if sys.version_info < MIN_REQ_PYTHON:\n",
-    "    print(\"Check the Kernel->Change Kernel menu and ensure that Python 3.6\")\n",
-    "    print(\"or later is selected as the active kernel.\")\n",
-    "    sys.exit(\"Python %s.%s or later is required.\\n\" % MIN_REQ_PYTHON)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": 1,
@@ -105,7 +84,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Imports Complete\n"
+      "Imports Complete - msticpy 3.0.2\n"
      ]
     }
    ],
@@ -114,11 +93,12 @@
     "from datetime import datetime, timedelta\n",
     "\n",
     "import pandas as pd\n",
+    "import msticpy\n",
     "\n",
     "# data library imports\n",
-    "from msticpy.data.data_providers import QueryProvider\n",
+    "from msticpy.data.core.data_providers import QueryProvider\n",
     "\n",
-    "print(\"Imports Complete\")"
+    "print(f\"Imports Complete - msticpy {msticpy.__version__}\")"
    ]
   },
   {
@@ -143,22 +123,6 @@
     "# os.environ['MSTICPYCONFIG'] = '/path/to/msticpyconfig.yaml'"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# FIXME! does not get MSTICPYCONFIG...\n",
-    "from msticpy.config import MpConfigEdit\n",
-    "\n",
-    "# mpconfig = MpConfigFile()\n",
-    "# mpconfig.load_default()\n",
-    "# mpconfig.view_settings()\n",
-    "mpedit = MpConfigEdit()\n",
-    "mpedit"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -167,7 +131,9 @@
     "## Instantiating a query provider\n",
     "\n",
     "You can instantiate a data provider for OpenObserve by specifying the credentials in connect or in msticpy config file. \n",
-    "<br> If the details are correct and authentication is successful, it will show connected."
+    "<br> If the details are correct and authentication is successful, it will show connected.\n",
+    "\n",
+    "Warning! url should not finish with a trailing slash, else it may trigger some security mechanism and return 401 error."
    ]
   },
   {
@@ -370,7 +336,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 63,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [

docs/source/data_acquisition/DataProv-OpenObserve.rst

@@ -0,0 +1,114 @@
+OpenObserve Provider
+==================
+
+OpenObserve Configuration
+-----------------------
+
+You can store your connection details in *msticpyconfig.yaml*.
+
+For more information on using and configuring *msticpyconfig.yaml* see
+:doc:`msticpy Package Configuration <../getting_started/msticpyconfig>`
+and :doc:`MSTICPy Settings Editor<../getting_started/SettingsEditor>`
+
+The settings in the file should look like the following:
+
+.. code:: yaml
+
+    DataProviders:
+      OpenObserve:
+        Args:
+          connection_str: openobserve_url
+          user:
+          password:
+
+We strongly recommend storing the password value
+in Azure Key Vault. You can replace the text value with a referenced
+to a Key Vault secret using the MSTICPy configuration editor.
+
+Your configuration when using Key Vault should look like the following:
+
+.. code:: yaml
+
+    DataProviders:
+      OpenObserve:
+        Args:
+          connection_str: openobserve_url
+          user:
+          password:
+            KeyVault:
+
+Loading a QueryProvider for OpenObserve
+-------------------------------------------
+
+.. code:: ipython3
+
+        qry_prov = QueryProvider("OpenObserve")
+
+
+Connecting to OpenObserve
+-----------------------------
+
+The parameters required for connection to OpenObserve can be passed in
+a number of ways. The simplest is to configure your settings
+in msticpyconfig. You can then just call connect with no parameters.
+
+Alternatively, you can pass the required connection parameters
+to the driver as parameters to the driver.
+
+.. code:: ipython3
+
+        qry_prov.connect()
+
+
+If you have configured multiple instances you must specify
+an instance name when you call connect.
+
+.. code:: ipython3
+
+        qry_prov.connect(instance="Instance2")
+
+Running a OpenObserve query
+-------------------------
+
+OpenObserve supports a number of optional query time parameters.
+Details of those parameters can be found here
+:py:meth:`msticpy.data.drivers.openobserve_driver.query`
+
+Be mindful that there is no standard schema by default in openobserve
+and streams (aka table) naming is depending on setup choice.
+Review corresponding streams before digging further.
+Also know, that by default only a _timestamp field matching ingestion
+or received time exists. the logs or message time must be extracted
+through pipelines.
+
+.. code:: ipython3
+
+    df_streams = qry_prov.list_streams()
+    df_streams[df_streams['stream_type'] == 'logs'][['name']].head()
+
+    query = """SELECT host_name as "host_name",
+       min(_timestamp) as "firstseen",
+       max(_timestamp) as "lastseen",
+       count() as "count"
+       FROM "journald"  GROUP BY host_name
+    """
+    df = qry_prov.exec_query(query, days=1, verbosity=3)
+    df.head()
+
+.. code:: ipython3
+
+    query = """SELECT..."""
+    df = qry_prov.exec_query(
+        query,
+        start=datetime.now() - timedelta(days=6.001),
+        end=datetime.now() - timedelta(days=6)
+    )
+    df.head()
+
+Other OpenObserve Documentation
+-----------------------------
+
+For examples of using the OpenObserve provider, see the sample
+`OpenObserve Notebook <https://github.com/microsoft/msticpy/blob/main/docs/notebooks/OpenObserve-DataConnector.ipynb>`
+
+:py:mod:`OpenObserve driver API documentation<msticpy.data.drivers.openobserve_driver>`

msticpy/data/drivers/openobserve_driver.py

@@ -318,6 +318,8 @@ def query(
             file path for exporte results.
         time_columns: array[string]
             returning columns which format should be dataframe timestamp
+        time_unit: string
+            pandas to_datetime unit argument, usually 'us' for epoch microseconds
         numeric_columns: array[string]
             returning columns which format should be dataframe numeric
 
@@ -335,6 +337,7 @@ def query(
         exporting = kwargs.pop("exporting", False)
         export_path = kwargs.pop("export_path", "")
         time_columns = kwargs.pop("time_columns", [])
+        time_unit = kwargs.pop("time_unit", "")
         numeric_columns = kwargs.pop("numeric_columns", [])
 
         dataframe_res = self._query(query, **kwargs)
@@ -351,7 +354,9 @@ def query(
                 if col in numeric_columns:
                     dataframe_res[col] = pd.to_numeric(dataframe_res[col])
                 # ensure timestamp format
-                if col in ["_timestamp"] + time_columns:
+                if col in ["_timestamp"] + time_columns and time_unit != "":
+                    dataframe_res[col] = pd.to_datetime(dataframe_res[col], unit=time_unit)
+                if col in ["_timestamp"] + time_columns and time_unit == "":
                     dataframe_res[col] = pd.to_datetime(dataframe_res[col])
 
             except Exception as err:
@@ -423,3 +428,41 @@ def _get_openobserve_settings(
             openobserve_settings = sl_settings.get("OpenObserve")
             is_instance_name = False
         return getattr(openobserve_settings, "args", {}), is_instance_name
+
+    def list_streams(self) -> tuple[pd.DataFrame, Any]:
+        """
+        List streams (aka available tables) and return DataFrame of results.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        tuple[pd.DataFrame, Any]
+            A DataFrame (if successful) or
+            the underlying provider result if an error occurs.
+
+        """
+        self._ensure_connected()
+        df_streams = self.service.list_objects2df("streams")
+        return df_streams
+
+    def list_alerts(self) -> tuple[pd.DataFrame, Any]:
+        """
+        List alerts and return DataFrame of results.
+
+        Parameters
+        ----------
+        None
+
+        Returns
+        -------
+        tuple[pd.DataFrame, Any]
+            A DataFrame (if successful) or
+            the underlying provider result if an error occurs.
+
+        """
+        self._ensure_connected()
+        df_alerts = self.service.list_objects2df("alerts")
+        return df_alerts