From d08b5d09970e68c03720cc27bf968862b98c8b46 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 13 Sep 2026 20:09:57 +0000 Subject: [PATCH 1/8] feat: add query() time_unit option, list_streams(), list_alerts() --- msticpy/data/drivers/openobserve_driver.py | 46 +++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/msticpy/data/drivers/openobserve_driver.py b/msticpy/data/drivers/openobserve_driver.py index b6dfb82d..f1eb4f31 100644 --- a/msticpy/data/drivers/openobserve_driver.py +++ b/msticpy/data/drivers/openobserve_driver.py @@ -204,7 +204,8 @@ def _query( """ del query_source - self._ensure_connected() + if not self._connected: + raise self._create_not_connected_err("OpenObserve") verbosity = kwargs.pop("verbosity", 0) timezone = kwargs.pop("timezone", "UTC") @@ -318,6 +319,8 @@ def query( file path for exporte results. time_columns: array[string] returning columns which format should be dataframe timestamp + time_unit: string + pandas to_datetime unit argument, usually 'us' for epoch microseconds numeric_columns: array[string] returning columns which format should be dataframe numeric @@ -335,6 +338,7 @@ def query( exporting = kwargs.pop("exporting", False) export_path = kwargs.pop("export_path", "") time_columns = kwargs.pop("time_columns", []) + time_unit = kwargs.pop("time_unit", "") numeric_columns = kwargs.pop("numeric_columns", []) dataframe_res = self._query(query, **kwargs) @@ -351,7 +355,9 @@ def query( if col in numeric_columns: dataframe_res[col] = pd.to_numeric(dataframe_res[col]) # ensure timestamp format - if col in ["_timestamp"] + time_columns: + if col in ["_timestamp"] + time_columns and time_unit != "": + dataframe_res[col] = pd.to_datetime(dataframe_res[col], unit=time_unit) + if col in ["_timestamp"] + time_columns and time_unit == "": dataframe_res[col] = pd.to_datetime(dataframe_res[col]) except Exception as err: @@ -423,3 +429,39 @@ def _get_openobserve_settings( openobserve_settings = sl_settings.get("OpenObserve") is_instance_name = False return getattr(openobserve_settings, "args", {}), is_instance_name + + def list_streams(self) -> tuple[pd.DataFrame, Any]: + """ + List streams (aka available tables) and return DataFrame of results. + + Parameters + ---------- + None + + Returns + ------- + tuple[pd.DataFrame, Any] + A DataFrame (if successful) or + the underlying provider result if an error occurs. + + """ + df_streams = self.service.list_objects2df("streams") + return df_streams + + def list_alerts(self) -> tuple[pd.DataFrame, Any]: + """ + List alerts and return DataFrame of results. + + Parameters + ---------- + None + + Returns + ------- + tuple[pd.DataFrame, Any] + A DataFrame (if successful) or + the underlying provider result if an error occurs. + + """ + df_alerts = self.service.list_objects2df("alerts") + return df_alerts From c3d8e82b10bfbbfd23ac0bfcd04ab09514d273e8 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 13 Sep 2026 20:14:41 +0000 Subject: [PATCH 2/8] docs: fix OpenObserve example notebook --- docs/notebooks/Openobserve-DataConnector.ipynb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/notebooks/Openobserve-DataConnector.ipynb b/docs/notebooks/Openobserve-DataConnector.ipynb index d0d28374..3195d473 100644 --- a/docs/notebooks/Openobserve-DataConnector.ipynb +++ b/docs/notebooks/Openobserve-DataConnector.ipynb @@ -58,6 +58,7 @@ }, { "cell_type": "markdown", + "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2020-08-07T17:50:18.361039Z", @@ -82,7 +83,9 @@ "outputs": [ { "name": "stdout", - "output_type": "stream" + "output_type": "stream", + "text": [ + ] } ], "source": [ From 4f148995c4f271598fcc53656a6602333b26e8a4 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 13 Sep 2026 20:15:46 +0000 Subject: [PATCH 3/8] docs: add OpenObserve provider --- .../data_acquisition/DataProv-OpenObserve.rst | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 docs/source/data_acquisition/DataProv-OpenObserve.rst diff --git a/docs/source/data_acquisition/DataProv-OpenObserve.rst b/docs/source/data_acquisition/DataProv-OpenObserve.rst new file mode 100644 index 00000000..5b3d0c28 --- /dev/null +++ b/docs/source/data_acquisition/DataProv-OpenObserve.rst @@ -0,0 +1,116 @@ +OpenObserve Provider +================== + +OpenObserve Configuration +----------------------- + +You can store your connection details in *msticpyconfig.yaml*. + +For more information on using and configuring *msticpyconfig.yaml* see +:doc:`msticpy Package Configuration <../getting_started/msticpyconfig>` +and :doc:`MSTICPy Settings Editor<../getting_started/SettingsEditor>` + +The settings in the file should look like the following: + +.. code:: yaml + + DataProviders: + OpenObserve: + Args: + connection_str: openobserve_url + user: + password: + +We strongly recommend storing the password value +in Azure Key Vault. You can replace the text value with a referenced +to a Key Vault secret using the MSTICPy configuration editor. + +Your configuration when using Key Vault should look like the following: + +.. code:: yaml + + DataProviders: + OpenObserve: + Args: + connection_str: openobserve_url + user: + password: + KeyVault: + +Loading a QueryProvider for OpenObserve +------------------------------------------- + +.. code:: ipython3 + + qry_prov = QueryProvider("OpenObserve") + + +Connecting to OpenObserve +----------------------------- + +The parameters required for connection to OpenObserve can be passed in +a number of ways. The simplest is to configure your settings +in msticpyconfig. You can then just call connect with no parameters. + +Alternatively, you can pass the required connection parameters +to the driver as parameters to the driver. + +.. code:: ipython3 + + qry_prov.connect() + + +If you have configured multiple instances you must specify +an instance name when you call connect. + +.. code:: ipython3 + + qry_prov.connect(instance="Tenant2") + +Running a OpenObserve query +------------------------- + +OpenObserve supports a number of optional query time parameters. +Details of those parameters can be found here +:py:meth:`msticpy.data.drivers.openobserve_driver.query` + +Be mindful that there is no standard schema by default in openobserve +and streams (aka table) naming is depending on setup choice. +Review corresponding streams before digging further. +Also know, that by default only a _timestamp field matching ingestion +or received time exists. the logs or message time must be extracted +through pipelines. + +.. code:: ipython3 + + df_streams = openobserve_prov.list_streams() + df_streams[df_streams['stream_type'] == 'logs'][['name']].head() + + query = """SELECT host_name as "host_name", + min(_timestamp) as "firstseen", + max(_timestamp) as "lastseen", + count() as "count" + FROM "journald" GROUP BY host_name + """" + df = openobserve_prov.exec_query(query, days=1, verbosity=3) + df.head() + +.. code:: ipython3 + + query = """SELECT... + df = openobserve_prov.exec_query( + query, + start=datetime.now() - timedelta(days=6.001), + end=datetime.now() - timedelta(days=6) + ) + df.head() + +Other OpenObserve Documentation +----------------------------- + +For examples of using the OpenObserve provider, see the sample +`OpenObserve Notebook ` + +Built-in :ref:`data_acquisition/DataQueries:Queries for OpenObserve`. + +:py:mod:`OpenObserve driver API documentation` From e5f5097a7c88bfe6cf54007dc8923e8590bce287 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 27 Sep 2026 19:42:36 +0000 Subject: [PATCH 4/8] fix: add/restore _ensure_connected() --- msticpy/data/drivers/openobserve_driver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/msticpy/data/drivers/openobserve_driver.py b/msticpy/data/drivers/openobserve_driver.py index f1eb4f31..92efe877 100644 --- a/msticpy/data/drivers/openobserve_driver.py +++ b/msticpy/data/drivers/openobserve_driver.py @@ -204,8 +204,7 @@ def _query( """ del query_source - if not self._connected: - raise self._create_not_connected_err("OpenObserve") + self._ensure_connected() verbosity = kwargs.pop("verbosity", 0) timezone = kwargs.pop("timezone", "UTC") @@ -445,6 +444,7 @@ def list_streams(self) -> tuple[pd.DataFrame, Any]: the underlying provider result if an error occurs. """ + self._ensure_connected() df_streams = self.service.list_objects2df("streams") return df_streams @@ -463,5 +463,6 @@ def list_alerts(self) -> tuple[pd.DataFrame, Any]: the underlying provider result if an error occurs. """ + self._ensure_connected() df_alerts = self.service.list_objects2df("alerts") return df_alerts From 630287f82cfdd4ebae98bc564f674ea1945a50b1 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 27 Sep 2026 19:42:55 +0000 Subject: [PATCH 5/8] docs: fix code examples --- .../data_acquisition/DataProv-OpenObserve.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-OpenObserve.rst b/docs/source/data_acquisition/DataProv-OpenObserve.rst index 5b3d0c28..6d9e9824 100644 --- a/docs/source/data_acquisition/DataProv-OpenObserve.rst +++ b/docs/source/data_acquisition/DataProv-OpenObserve.rst @@ -65,7 +65,7 @@ an instance name when you call connect. .. code:: ipython3 - qry_prov.connect(instance="Tenant2") + qry_prov.connect(instance="Instance2") Running a OpenObserve query ------------------------- @@ -83,7 +83,7 @@ through pipelines. .. code:: ipython3 - df_streams = openobserve_prov.list_streams() + df_streams = qry_prov.list_streams() df_streams[df_streams['stream_type'] == 'logs'][['name']].head() query = """SELECT host_name as "host_name", @@ -91,14 +91,14 @@ through pipelines. max(_timestamp) as "lastseen", count() as "count" FROM "journald" GROUP BY host_name - """" - df = openobserve_prov.exec_query(query, days=1, verbosity=3) + """ + df = qry_prov.exec_query(query, days=1, verbosity=3) df.head() .. code:: ipython3 - query = """SELECT... - df = openobserve_prov.exec_query( + query = """SELECT...""" + df = qry_prov.exec_query( query, start=datetime.now() - timedelta(days=6.001), end=datetime.now() - timedelta(days=6) @@ -109,7 +109,7 @@ Other OpenObserve Documentation ----------------------------- For examples of using the OpenObserve provider, see the sample -`OpenObserve Notebook ` +`OpenObserve Notebook ` Built-in :ref:`data_acquisition/DataQueries:Queries for OpenObserve`. From 208dc95ae7e40f345f02196034d3180a19585490 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 27 Sep 2026 19:44:24 +0000 Subject: [PATCH 6/8] docs: fix openobserve notebook - WIP: line 91 has source --- .../notebooks/Openobserve-DataConnector.ipynb | 41 ++++--------------- 1 file changed, 8 insertions(+), 33 deletions(-) diff --git a/docs/notebooks/Openobserve-DataConnector.ipynb b/docs/notebooks/Openobserve-DataConnector.ipynb index 3195d473..8962c864 100644 --- a/docs/notebooks/Openobserve-DataConnector.ipynb +++ b/docs/notebooks/Openobserve-DataConnector.ipynb @@ -58,7 +58,6 @@ }, { "cell_type": "markdown", - "execution_count": 1, "metadata": { "ExecuteTime": { "end_time": "2020-08-07T17:50:18.361039Z", @@ -87,16 +86,6 @@ "text": [ ] } - ], - "source": [ - "# Check we are running Python 3.6\n", - "import sys\n", - "\n", - "MIN_REQ_PYTHON = (3, 6)\n", - "if sys.version_info < MIN_REQ_PYTHON:\n", - " print(\"Check the Kernel->Change Kernel menu and ensure that Python 3.6\")\n", - " print(\"or later is selected as the active kernel.\")\n", - " sys.exit(\"Python %s.%s or later is required.\\n\" % MIN_REQ_PYTHON)" ] }, { @@ -108,7 +97,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Imports Complete\n" + "Imports Complete - msticpy 3.0.2\n" ] } ], @@ -117,11 +106,11 @@ "from datetime import datetime, timedelta\n", "\n", "import pandas as pd\n", - "\n", + "import msticpy\n", "# data library imports\n", - "from msticpy.data.data_providers import QueryProvider\n", + "from msticpy.data.core.data_providers import QueryProvider\n", "\n", - "print(\"Imports Complete\")" + "print(f\"Imports Complete - msticpy {msticpy.__version__}\")" ] }, { @@ -146,22 +135,6 @@ "# os.environ['MSTICPYCONFIG'] = '/path/to/msticpyconfig.yaml'" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME! does not get MSTICPYCONFIG...\n", - "from msticpy.config import MpConfigEdit\n", - "\n", - "# mpconfig = MpConfigFile()\n", - "# mpconfig.load_default()\n", - "# mpconfig.view_settings()\n", - "mpedit = MpConfigEdit()\n", - "mpedit" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -170,7 +143,9 @@ "## Instantiating a query provider\n", "\n", "You can instantiate a data provider for OpenObserve by specifying the credentials in connect or in msticpy config file. \n", - "
If the details are correct and authentication is successful, it will show connected." + "
If the details are correct and authentication is successful, it will show connected.\n", + "\n", + "Warning! url should not finish with a trailing slash, else it may trigger some security mechanism and return 401 error." ] }, { @@ -373,7 +348,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ From b17fc08eb49e5e9bfda99ad002b3646b46e198ba Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 27 Sep 2026 19:46:09 +0000 Subject: [PATCH 7/8] docs: remove non-existing sphinx ref --- docs/source/data_acquisition/DataProv-OpenObserve.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/data_acquisition/DataProv-OpenObserve.rst b/docs/source/data_acquisition/DataProv-OpenObserve.rst index 6d9e9824..a3de187f 100644 --- a/docs/source/data_acquisition/DataProv-OpenObserve.rst +++ b/docs/source/data_acquisition/DataProv-OpenObserve.rst @@ -111,6 +111,4 @@ Other OpenObserve Documentation For examples of using the OpenObserve provider, see the sample `OpenObserve Notebook ` -Built-in :ref:`data_acquisition/DataQueries:Queries for OpenObserve`. - :py:mod:`OpenObserve driver API documentation` From e654506664ad9c0f7f606de9b7b5775a8f8ba403 Mon Sep 17 00:00:00 2001 From: juju4 Date: Sun, 27 Sep 2026 19:52:11 +0000 Subject: [PATCH 8/8] docs: fix openobserve notebook (2) --- docs/notebooks/Openobserve-DataConnector.ipynb | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/docs/notebooks/Openobserve-DataConnector.ipynb b/docs/notebooks/Openobserve-DataConnector.ipynb index 8962c864..0f6f1a7d 100644 --- a/docs/notebooks/Openobserve-DataConnector.ipynb +++ b/docs/notebooks/Openobserve-DataConnector.ipynb @@ -75,19 +75,6 @@ "```" ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - ] - } - ] - }, { "cell_type": "code", "execution_count": 1, @@ -107,6 +94,7 @@ "\n", "import pandas as pd\n", "import msticpy\n", + "\n", "# data library imports\n", "from msticpy.data.core.data_providers import QueryProvider\n", "\n",