diff --git a/docs/notebooks/Openobserve-DataConnector.ipynb b/docs/notebooks/Openobserve-DataConnector.ipynb index d0d28374..0f6f1a7d 100644 --- a/docs/notebooks/Openobserve-DataConnector.ipynb +++ b/docs/notebooks/Openobserve-DataConnector.ipynb @@ -75,27 +75,6 @@ "```" ] }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream" - } - ], - "source": [ - "# Check we are running Python 3.6\n", - "import sys\n", - "\n", - "MIN_REQ_PYTHON = (3, 6)\n", - "if sys.version_info < MIN_REQ_PYTHON:\n", - " print(\"Check the Kernel->Change Kernel menu and ensure that Python 3.6\")\n", - " print(\"or later is selected as the active kernel.\")\n", - " sys.exit(\"Python %s.%s or later is required.\\n\" % MIN_REQ_PYTHON)" - ] - }, { "cell_type": "code", "execution_count": 1, @@ -105,7 +84,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Imports Complete\n" + "Imports Complete - msticpy 3.0.2\n" ] } ], @@ -114,11 +93,12 @@ "from datetime import datetime, timedelta\n", "\n", "import pandas as pd\n", + "import msticpy\n", "\n", "# data library imports\n", - "from msticpy.data.data_providers import QueryProvider\n", + "from msticpy.data.core.data_providers import QueryProvider\n", "\n", - "print(\"Imports Complete\")" + "print(f\"Imports Complete - msticpy {msticpy.__version__}\")" ] }, { @@ -143,22 +123,6 @@ "# os.environ['MSTICPYCONFIG'] = '/path/to/msticpyconfig.yaml'" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# FIXME! does not get MSTICPYCONFIG...\n", - "from msticpy.config import MpConfigEdit\n", - "\n", - "# mpconfig = MpConfigFile()\n", - "# mpconfig.load_default()\n", - "# mpconfig.view_settings()\n", - "mpedit = MpConfigEdit()\n", - "mpedit" - ] - }, { "cell_type": "markdown", "metadata": {}, @@ -167,7 +131,9 @@ "## Instantiating a query provider\n", "\n", "You can instantiate a data provider for OpenObserve by specifying the credentials in connect or in msticpy config file. \n", - "
If the details are correct and authentication is successful, it will show connected." + "
If the details are correct and authentication is successful, it will show connected.\n", + "\n", + "Warning! url should not finish with a trailing slash, else it may trigger some security mechanism and return 401 error." ] }, { @@ -370,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 63, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ diff --git a/docs/source/data_acquisition/DataProv-OpenObserve.rst b/docs/source/data_acquisition/DataProv-OpenObserve.rst new file mode 100644 index 00000000..a3de187f --- /dev/null +++ b/docs/source/data_acquisition/DataProv-OpenObserve.rst @@ -0,0 +1,114 @@ +OpenObserve Provider +================== + +OpenObserve Configuration +----------------------- + +You can store your connection details in *msticpyconfig.yaml*. + +For more information on using and configuring *msticpyconfig.yaml* see +:doc:`msticpy Package Configuration <../getting_started/msticpyconfig>` +and :doc:`MSTICPy Settings Editor<../getting_started/SettingsEditor>` + +The settings in the file should look like the following: + +.. code:: yaml + + DataProviders: + OpenObserve: + Args: + connection_str: openobserve_url + user: + password: + +We strongly recommend storing the password value +in Azure Key Vault. You can replace the text value with a referenced +to a Key Vault secret using the MSTICPy configuration editor. + +Your configuration when using Key Vault should look like the following: + +.. code:: yaml + + DataProviders: + OpenObserve: + Args: + connection_str: openobserve_url + user: + password: + KeyVault: + +Loading a QueryProvider for OpenObserve +------------------------------------------- + +.. code:: ipython3 + + qry_prov = QueryProvider("OpenObserve") + + +Connecting to OpenObserve +----------------------------- + +The parameters required for connection to OpenObserve can be passed in +a number of ways. The simplest is to configure your settings +in msticpyconfig. You can then just call connect with no parameters. + +Alternatively, you can pass the required connection parameters +to the driver as parameters to the driver. + +.. code:: ipython3 + + qry_prov.connect() + + +If you have configured multiple instances you must specify +an instance name when you call connect. + +.. code:: ipython3 + + qry_prov.connect(instance="Instance2") + +Running a OpenObserve query +------------------------- + +OpenObserve supports a number of optional query time parameters. +Details of those parameters can be found here +:py:meth:`msticpy.data.drivers.openobserve_driver.query` + +Be mindful that there is no standard schema by default in openobserve +and streams (aka table) naming is depending on setup choice. +Review corresponding streams before digging further. +Also know, that by default only a _timestamp field matching ingestion +or received time exists. the logs or message time must be extracted +through pipelines. + +.. code:: ipython3 + + df_streams = qry_prov.list_streams() + df_streams[df_streams['stream_type'] == 'logs'][['name']].head() + + query = """SELECT host_name as "host_name", + min(_timestamp) as "firstseen", + max(_timestamp) as "lastseen", + count() as "count" + FROM "journald" GROUP BY host_name + """ + df = qry_prov.exec_query(query, days=1, verbosity=3) + df.head() + +.. code:: ipython3 + + query = """SELECT...""" + df = qry_prov.exec_query( + query, + start=datetime.now() - timedelta(days=6.001), + end=datetime.now() - timedelta(days=6) + ) + df.head() + +Other OpenObserve Documentation +----------------------------- + +For examples of using the OpenObserve provider, see the sample +`OpenObserve Notebook ` + +:py:mod:`OpenObserve driver API documentation` diff --git a/msticpy/data/drivers/openobserve_driver.py b/msticpy/data/drivers/openobserve_driver.py index b6dfb82d..92efe877 100644 --- a/msticpy/data/drivers/openobserve_driver.py +++ b/msticpy/data/drivers/openobserve_driver.py @@ -318,6 +318,8 @@ def query( file path for exporte results. time_columns: array[string] returning columns which format should be dataframe timestamp + time_unit: string + pandas to_datetime unit argument, usually 'us' for epoch microseconds numeric_columns: array[string] returning columns which format should be dataframe numeric @@ -335,6 +337,7 @@ def query( exporting = kwargs.pop("exporting", False) export_path = kwargs.pop("export_path", "") time_columns = kwargs.pop("time_columns", []) + time_unit = kwargs.pop("time_unit", "") numeric_columns = kwargs.pop("numeric_columns", []) dataframe_res = self._query(query, **kwargs) @@ -351,7 +354,9 @@ def query( if col in numeric_columns: dataframe_res[col] = pd.to_numeric(dataframe_res[col]) # ensure timestamp format - if col in ["_timestamp"] + time_columns: + if col in ["_timestamp"] + time_columns and time_unit != "": + dataframe_res[col] = pd.to_datetime(dataframe_res[col], unit=time_unit) + if col in ["_timestamp"] + time_columns and time_unit == "": dataframe_res[col] = pd.to_datetime(dataframe_res[col]) except Exception as err: @@ -423,3 +428,41 @@ def _get_openobserve_settings( openobserve_settings = sl_settings.get("OpenObserve") is_instance_name = False return getattr(openobserve_settings, "args", {}), is_instance_name + + def list_streams(self) -> tuple[pd.DataFrame, Any]: + """ + List streams (aka available tables) and return DataFrame of results. + + Parameters + ---------- + None + + Returns + ------- + tuple[pd.DataFrame, Any] + A DataFrame (if successful) or + the underlying provider result if an error occurs. + + """ + self._ensure_connected() + df_streams = self.service.list_objects2df("streams") + return df_streams + + def list_alerts(self) -> tuple[pd.DataFrame, Any]: + """ + List alerts and return DataFrame of results. + + Parameters + ---------- + None + + Returns + ------- + tuple[pd.DataFrame, Any] + A DataFrame (if successful) or + the underlying provider result if an error occurs. + + """ + self._ensure_connected() + df_alerts = self.service.list_objects2df("alerts") + return df_alerts