From 70dc06a46b009743a5e72d2d1121de6e07c0352a Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Tue, 12 Nov 2024 20:13:13 +0000
Subject: [PATCH 01/13] Document read_zeek function

---
 lib/aip/data/functions.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index e13b3e4..e25155c 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -17,14 +17,23 @@
 }
 
 def read_zeek(path, **kwargs):
+    """
+    Reads a Zeek file to a DataFrame
+    """
     try:
+        # Load Zeek path to DataFrame
         df = z2p.read_zeek(path, **kwargs)
+
+        # Convert to readable time format
         if 'ts' in df.keys():
             df['ts'] = pd.to_datetime(df.ts, unit='s')
+
+        # Returns data frame
         return df
     except:
         raise z2p.NotAZeekFile(path)
 
+
 def read_argus(path, **kwargs):
     from os import path as ospath
     if ospath.exists(path.path + '.csv'):

From 7a71e8b065bf3f141c01a9073f97c1afeaa1e799 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Tue, 12 Nov 2024 20:18:40 +0000
Subject: [PATCH 02/13] Split imports for clarity and control

---
 lib/aip/data/access.py    | 11 ++++++++---
 lib/aip/data/functions.py | 11 ++++++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 67cc325..7613896 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -36,10 +36,15 @@
 import pandas as pd
 import numpy as np
 
-from aip.data.functions import scramble, read_zeek, getrawdata, removerawdata
-from joblib import Parallel, delayed
-from os import scandir, path
+from joblib import Parallel
+from joblib import delayed
+from os import scandir
+from os import path
 from pathlib import Path
+from aip.data.functions import scramble
+from aip.data.functions import read_zeek
+from aip.data.functions import getrawdata
+from aip.data.functions import removerawdata
 
 project_dir = Path(__file__).resolve().parents[3]
 
diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index e25155c..2f818d9 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -3,12 +3,17 @@
 import hashlib
 import pandas as pd
 import shutil
-import subprocess, shlex
+import shlex
+import subprocess
 import zeeklog2pandas as z2p
 
 from dotenv import dotenv_values
-from joblib import Parallel, delayed
-from os import makedirs, path, access, W_OK
+from joblib import Parallel
+from joblib import delayed
+from os import makedirs
+from os import path
+from os import access
+from os import W_OK
 from pathlib import Path
 
 _project_dir = Path(__file__).resolve().parents[3]

From 481581d2f4a5bc1c31b54e4e20d8250a589fbfc3 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 13:10:45 +0000
Subject: [PATCH 03/13] Document getrawdata()

---
 lib/aip/data/functions.py | 36 ++++++++++++++++++++++++++++++------
 1 file changed, 30 insertions(+), 6 deletions(-)

diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index 2f818d9..0b8379e 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -21,6 +21,7 @@
     **dotenv_values(path.join(_project_dir, ".env")),  # load sensitive variables
 }
 
+
 def read_zeek(path, **kwargs):
     """
     Reads a Zeek file to a DataFrame
@@ -39,6 +40,7 @@ def read_zeek(path, **kwargs):
         raise z2p.NotAZeekFile(path)
 
 
+# Currently deprecated, AIP is using now strictly Zeek logs
 def read_argus(path, **kwargs):
     from os import path as ospath
     if ospath.exists(path.path + '.csv'):
@@ -54,16 +56,39 @@ def read_argus(path, **kwargs):
     df.rename(columns={'StartTime':'ts', 'SrcAddr':'id.orig_h', 'DstAddr':'id.resp_h', 'Dur': 'duration', 'SrcBytes': 'orig_ip_bytes', 'SrcPkts': 'orig_pkts'}, inplace=True)
     return df
 
+
+# This function is unused right now
 def scramble(s):
     return hashlib.sha1(_config['salt'].encode() + s.encode()).hexdigest()
 
+
 def getrawdata(date):
+    """
+    Retrieves Zeek data from a remote? location and stores it
+    on a directory for AIP to process it. The copy is done in
+    parallel.
+    """
+
+    # Validate date is well formatted
     dt.datetime.strptime(date, '%Y-%m-%d')
-    p = path.join(_project_dir,'data','raw', date)
-    if access(p, W_OK):
-        makedirs(p, exist_ok=True)
-        commands = [shlex.split(_config['magic'] + f'{date}/conn.{x:02}* ' + p) for x in range(0,24)]
-        Parallel(n_jobs=24, backend='threading')(delayed(subprocess.run)(c) for c in commands)
+
+    raw_data_dir = path.join(_project_dir,'data','raw', date)
+
+    # Ensure directory exists and is writable
+    if access(raw_data_dir, W_OK):
+        # Create directory, ignore if it exists
+        makedirs(raw_data_dir, exist_ok=True)
+
+        # The next part seems to be prepared to retrieve data from a location
+        # and store it in the data/raw/YYYY-MM-DD directory for processing.
+        commands = [
+            shlex.split(_config['magic'] + f'{date}/conn.{x:02}* ' + raw_data_dir) 
+            for x in range(0,24)
+        ]
+
+        # Attempting to run the previous commands in parallel
+        Parallel(n_jobs=24, backend='threading')(delayed(subprocess.run)(cmd) for cmd in commands)
+
 
 def removerawdata(date, force=False):
     dt.datetime.strptime(date, '%Y-%m-%d')
@@ -71,4 +96,3 @@ def removerawdata(date, force=False):
     # Only delete raw data if explicitly allowed in the configuration file
     if (_config['remove_raw_data'].lower() == 'true') or force:
         shutil.rmtree(p)
-

From cf3debc4cc3f2e5d72b033ded5b6b08d517cd5f3 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 13:32:46 +0000
Subject: [PATCH 04/13] Document removerawdata()

---
 lib/aip/data/functions.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index 0b8379e..65bfd36 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -91,8 +91,16 @@ def getrawdata(date):
 
 
 def removerawdata(date, force=False):
+    """
+    Remove (delete) the content of the raw data directory 
+    for a given date.
+    """
+    # Validate date is well formatted
     dt.datetime.strptime(date, '%Y-%m-%d')
-    p = path.join(_project_dir,'data','raw', date)
+
+    raw_data_dir = path.join(_project_dir,'data','raw', date)
+
     # Only delete raw data if explicitly allowed in the configuration file
     if (_config['remove_raw_data'].lower() == 'true') or force:
-        shutil.rmtree(p)
+        shutil.rmtree(raw_data_dir)
+

From b684c8a3f1b16f068640523ad5ceb4bb09b49711 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 13:47:06 +0000
Subject: [PATCH 05/13] Rename functions to follow naming convention

---
 lib/aip/data/access.py    | 6 +++---
 lib/aip/data/functions.py | 4 ++--
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 7613896..77c192b 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -43,8 +43,8 @@
 from pathlib import Path
 from aip.data.functions import scramble
 from aip.data.functions import read_zeek
-from aip.data.functions import getrawdata
-from aip.data.functions import removerawdata
+from aip.data.functions import get_raw_data
+from aip.data.functions import remove_raw_data
 
 project_dir = Path(__file__).resolve().parents[3]
 
@@ -107,7 +107,7 @@ def _process_raw_files(date):
     # if data directory does not exist, execute the magic to get it
     if path.isdir(path.join(project_dir,'data','raw', date)) == False:
         logging.debug(f'Downloading data for {date}')
-        getrawdata(date)
+        get_raw_data(date)
     # after this point, if directory does not exist, we can skip it.
     try:
         zeek_files = [x.path for x in scandir(path.join(project_dir,'data','raw', date)) if x.name.startswith('conn.')]
diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index 65bfd36..989fa45 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -62,7 +62,7 @@ def scramble(s):
     return hashlib.sha1(_config['salt'].encode() + s.encode()).hexdigest()
 
 
-def getrawdata(date):
+def get_raw_data(date):
     """
     Retrieves Zeek data from a remote? location and stores it
     on a directory for AIP to process it. The copy is done in
@@ -90,7 +90,7 @@ def getrawdata(date):
         Parallel(n_jobs=24, backend='threading')(delayed(subprocess.run)(cmd) for cmd in commands)
 
 
-def removerawdata(date, force=False):
+def remove_raw_data(date, force=False):
     """
     Remove (delete) the content of the raw data directory 
     for a given date.

From 46c0fc061890115b26faaaaf95ba0094e3f70681 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:12:33 +0000
Subject: [PATCH 06/13] Document _get_honeypot_ips

---
 lib/aip/data/access.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 77c192b..2708c16 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -53,26 +53,39 @@
 
 def _get_honeypot_ips(for_date=None):
     '''
-    Filter those honeypots active due date for_date, if there are operation dates in the honeypot file.
+    Filter active honeypots IPs due date for_date, if there are operation dates in the honeypot file.
+    The honeypots_public_ips.csv has the following format:
+        # List of IPs to look for to generate the attack files.
+        public_ip,operation_start_date,operation_end_date
     '''
     logger = logging.getLogger(__name__)
+
     # Check if the file exists before attempting to read it
     honeypot_public_ips = path.join(project_dir, 'data', 'external', 'honeypots_public_ips.csv')
 
+    # If the file does not exist raise an exception
     if not path.exists(honeypot_public_ips):
-        logger.error(f"File 'honeypot_public_ips.csv' does not exist. Raising error.")
-        raise FileNotFoundError("Required file 'honeypots_public_ips.csv' does not exist.")
+        raise FileNotFoundError("_get_honeypot_ips() required file 'honeypots_public_ips.csv' does not exist.")
 
+    # Read CSV located in data/external/honeypots_public_ips.csv
     honeypots = pd.read_csv(path.join(project_dir, 'data', 'external', 'honeypots_public_ips.csv'), comment='#')
+
     if for_date is not None:
+        # Convert to datetime object
         for_date = pd.to_datetime(for_date)
+
+        # Parsing start date
         if 'operation_start_date' in honeypots.keys():
             honeypots['operation_start_date'] = pd.to_datetime(honeypots['operation_start_date'])
+
+        # Parsing end date, filling emtpy values with date of 'today'
         if 'operation_end_date' in honeypots.keys():
             honeypots['operation_end_date'] = honeypots['operation_end_date'].fillna(dt.date.today())
             honeypots['operation_end_date'] = pd.to_datetime(honeypots['operation_end_date'])
+
         if ('operation_start_date' in honeypots.keys()) and 'operation_end_date' in honeypots.keys():
             honeypots = honeypots[(for_date >= honeypots['operation_start_date']) & (for_date <= honeypots['operation_end_date'])]
+
     ips = honeypots.public_ip.values
     return ips
 

From e4d49f6f7e6d212d58b197dcdca4791b286bffdf Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:23:48 +0000
Subject: [PATCH 07/13] Document which IPs are kept

---
 lib/aip/data/access.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 2708c16..46b613f 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -83,6 +83,7 @@ def _get_honeypot_ips(for_date=None):
             honeypots['operation_end_date'] = honeypots['operation_end_date'].fillna(dt.date.today())
             honeypots['operation_end_date'] = pd.to_datetime(honeypots['operation_end_date'])
 
+        # Keep honeypots active on the specified date
         if ('operation_start_date' in honeypots.keys()) and 'operation_end_date' in honeypots.keys():
             honeypots = honeypots[(for_date >= honeypots['operation_start_date']) & (for_date <= honeypots['operation_end_date'])]
 

From 83f77b4fafe882fba245f84ceaf788cb38f8d4d6 Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:26:21 +0000
Subject: [PATCH 08/13] Remove deprecated variable no longer used

---
 lib/aip/data/access.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 46b613f..42e3ae5 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -48,7 +48,6 @@
 
 project_dir = Path(__file__).resolve().parents[3]
 
-data_path = path.join(project_dir,'data')  # Deprecated, do not use
 data_dir = path.join(project_dir,'data')
 
 def _get_honeypot_ips(for_date=None):

From 4ecb2d7192f43241bfc741756d3935268c0948fc Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:31:31 +0000
Subject: [PATCH 09/13] Add PEP8 blank lines

---
 lib/aip/data/access.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 42e3ae5..bcc81fe 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -50,6 +50,7 @@
 
 data_dir = path.join(project_dir,'data')
 
+
 def _get_honeypot_ips(for_date=None):
     '''
     Filter active honeypots IPs due date for_date, if there are operation dates in the honeypot file.
@@ -89,6 +90,7 @@ def _get_honeypot_ips(for_date=None):
     ips = honeypots.public_ip.values
     return ips
 
+
 def _process_zeek_files(zeek_files, date):
     ips = _get_honeypot_ips()
     daily = pd.DataFrame()
@@ -100,6 +102,7 @@ def _process_zeek_files(zeek_files, date):
         daily = pd.concat([daily, hourly])
     return daily
 
+
 def _process_argus_files(argus_files, date):
     ips = _get_honeypot_ips()
     daily = pd.DataFrame()
@@ -111,6 +114,7 @@ def _process_argus_files(argus_files, date):
         daily = pd.concat([daily, hourly])
     return daily
 
+
 def _process_raw_files(date):
     '''
     Create a dataset for the date string date in the data/interim folder
@@ -137,6 +141,7 @@ def _process_raw_files(date):
     #removerawdata(date)
     return
 
+
 def _extract_attacks(date):
     '''
     Create a dataset for the date string date in the data/interim folder
@@ -171,6 +176,7 @@ def _extract_attacks(date):
     #removerawdata(date)
     return
 
+
 def process_zeek_files(dates=None):
     """ 
     Creates the dataset or part of it
@@ -188,6 +194,7 @@ def process_zeek_files(dates=None):
     Parallel(n_jobs=12, backend='multiprocessing')(delayed(_process_raw_files)(date) for date in dates)
     return
 
+
 def extract_attacks(dates=None):
     """
     Creates the dataset or part of it
@@ -211,6 +218,7 @@ def extract_attacks(dates=None):
     Parallel(n_jobs=12, backend='multiprocessing')(delayed(_extract_attacks)(date) for date in dates)
     return
 
+
 def get_attacks(start=None, end=None, dates=None, usecols=None):
     '''
     Returns a DataFrame with the attacks between the dates start and end or the
@@ -230,6 +238,7 @@ def get_attacks(start=None, end=None, dates=None, usecols=None):
             for date in dates]
     return dfs
 
+
 if __name__ == '__main__':
     log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s'
     #logging.basicConfig(level=logging.INFO, format=log_fmt)

From a93f2125778e47aaa6b161faca558444f81a004f Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:54:34 +0000
Subject: [PATCH 10/13] It seems this is used a lot, reintroducing

---
 lib/aip/data/access.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index bcc81fe..2cff4b0 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -48,6 +48,7 @@
 
 project_dir = Path(__file__).resolve().parents[3]
 
+data_path = path.join(project_dir,'data')  # Deprecated, do not use
 data_dir = path.join(project_dir,'data')
 
 

From 88d4e863d75f0c3ce80e198c8c1d334442ce534f Mon Sep 17 00:00:00 2001
From: verovaleros <vero.valeros@gmail.com>
Date: Wed, 13 Nov 2024 19:55:27 +0000
Subject: [PATCH 11/13] Document _process_zeek_files()

---
 lib/aip/data/access.py | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 2cff4b0..31fbf1b 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -92,15 +92,35 @@ def _get_honeypot_ips(for_date=None):
     return ips
 
 
-def _process_zeek_files(zeek_files, date):
+def _process_zeek_files(list_of_zeek_files, date):
+    """
+    Process a list of Zeek log files to extract all connections
+    from honeypot IPs for a given date.
+    """
+    # Retrieve the list of honeypot IPs
     ips = _get_honeypot_ips()
+
+    # Initialises daily, a dataframe that will contain
+    # all the connections from the honeypots IPs found
+    # on the input zeek files
     daily = pd.DataFrame()
-    for z in zeek_files:
+
+    # Process each zeek file in the input list
+    for zeek_file in list_of_zeek_files:
         hourly = pd.DataFrame()
-        zeekdata = read_zeek(z)
+
+        # Read the zeek file into a dataframe
+        zeekdata = read_zeek(zeek_file)
+
+        # Find all traffic from IPs on the zeek traffic 
         for ip in ips:
             hourly = pd.concat([hourly, zeekdata[zeekdata['id.resp_h'] == ip]])
+
+        # Store the hourly traffic on the daily dataframe
         daily = pd.concat([daily, hourly])
+
+    # Return a DF with all the traffic seen from the honeypot IPs
+    # on the input Zeek files
     return daily
 
 

From 1f3641908227f5be0d2989a4719403168dd144a7 Mon Sep 17 00:00:00 2001
From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com>
Date: Tue, 13 Oct 2026 12:08:05 +0200
Subject: [PATCH 12/13] Fix traffic direction in data access comments

---
 lib/aip/data/access.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/lib/aip/data/access.py b/lib/aip/data/access.py
index 31fbf1b..49ddb38 100644
--- a/lib/aip/data/access.py
+++ b/lib/aip/data/access.py
@@ -54,13 +54,12 @@
 
 def _get_honeypot_ips(for_date=None):
     '''
-    Filter active honeypots IPs due date for_date, if there are operation dates in the honeypot file.
-    The honeypots_public_ips.csv has the following format:
+    Return the public IPs of the honeypots. If for_date is given and the
+    honeypot file has operation dates, only return the honeypots active on
+    that date. The honeypots_public_ips.csv has the following format:
         # List of IPs to look for to generate the attack files.
         public_ip,operation_start_date,operation_end_date
     '''
-    logger = logging.getLogger(__name__)
-
     # Check if the file exists before attempting to read it
     honeypot_public_ips = path.join(project_dir, 'data', 'external', 'honeypots_public_ips.csv')
 
@@ -79,7 +78,7 @@ def _get_honeypot_ips(for_date=None):
         if 'operation_start_date' in honeypots.keys():
             honeypots['operation_start_date'] = pd.to_datetime(honeypots['operation_start_date'])
 
-        # Parsing end date, filling emtpy values with date of 'today'
+        # Parsing end date, filling empty values with date of 'today'
         if 'operation_end_date' in honeypots.keys():
             honeypots['operation_end_date'] = honeypots['operation_end_date'].fillna(dt.date.today())
             honeypots['operation_end_date'] = pd.to_datetime(honeypots['operation_end_date'])
@@ -95,13 +94,13 @@ def _get_honeypot_ips(for_date=None):
 def _process_zeek_files(list_of_zeek_files, date):
     """
     Process a list of Zeek log files to extract all connections
-    from honeypot IPs for a given date.
+    sent to the honeypot IPs (the attacks) for a given date.
     """
     # Retrieve the list of honeypot IPs
     ips = _get_honeypot_ips()
 
     # Initialises daily, a dataframe that will contain
-    # all the connections from the honeypots IPs found
+    # all the connections to the honeypot IPs found
     # on the input zeek files
     daily = pd.DataFrame()
 
@@ -112,14 +111,15 @@ def _process_zeek_files(list_of_zeek_files, date):
         # Read the zeek file into a dataframe
         zeekdata = read_zeek(zeek_file)
 
-        # Find all traffic from IPs on the zeek traffic 
+        # Keep the connections where the honeypot is the responder,
+        # that is, traffic sent to the honeypot
         for ip in ips:
             hourly = pd.concat([hourly, zeekdata[zeekdata['id.resp_h'] == ip]])
 
         # Store the hourly traffic on the daily dataframe
         daily = pd.concat([daily, hourly])
 
-    # Return a DF with all the traffic seen from the honeypot IPs
+    # Return a DF with all the traffic sent to the honeypot IPs
     # on the input Zeek files
     return daily
 
@@ -159,7 +159,7 @@ def _process_raw_files(date):
     daily.to_csv(path.join(project_dir,'data','interim', f'daily.conn.{date}.csv.gz'), index=False, compression='gzip')
     logger.debug('Writting file: ' + path.join(project_dir,'data','interim', f'daily.conn.{date}.csv.gz'))
     #logger.debug('Removing raw data (not needed anymore): ' + path.join(project_dir,'data','raw', f'{date}'))
-    #removerawdata(date)
+    #remove_raw_data(date)
     return
 
 
@@ -194,7 +194,7 @@ def _extract_attacks(date):
     logger.debug('Writting file: ' + path.join(project_dir,'data','processed', f'attacks.{date}.csv.gz'))
     df.to_csv(path.join(project_dir,'data','processed', f'attacks.{date}.csv.gz'), columns=['orig', 'flows', 'duration', 'packets', 'bytes'], index=False, compression='gzip')
     # logger.debug('Removing raw data (not needed anymore): ' + path.join(project_dir,'data','raw', f'{date}'))
-    #removerawdata(date)
+    #remove_raw_data(date)
     return
 
 

From e097e48079f85bdd143d0ff5a2218b050a969c84 Mon Sep 17 00:00:00 2001
From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com>
Date: Tue, 13 Oct 2026 12:08:05 +0200
Subject: [PATCH 13/13] Clarify raw data docstrings and note get_raw_data issue

---
 lib/aip/data/functions.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/lib/aip/data/functions.py b/lib/aip/data/functions.py
index 989fa45..a8f75d8 100644
--- a/lib/aip/data/functions.py
+++ b/lib/aip/data/functions.py
@@ -40,7 +40,7 @@ def read_zeek(path, **kwargs):
         raise z2p.NotAZeekFile(path)
 
 
-# Currently deprecated, AIP is using now strictly Zeek logs
+# Deprecated, AIP only reads Zeek logs
 def read_argus(path, **kwargs):
     from os import path as ospath
     if ospath.exists(path.path + '.csv'):
@@ -64,9 +64,13 @@ def scramble(s):
 
 def get_raw_data(date):
     """
-    Retrieves Zeek data from a remote? location and stores it
-    on a directory for AIP to process it. The copy is done in
-    parallel.
+    Copies the Zeek conn logs of a given date into data/raw/YYYY-MM-DD,
+    using the copy command set in 'magic' in the .env file (for example
+    'cp' or 'scp'). The 24 hourly files are copied in parallel.
+
+    Known issue: this function never copies anything. It is only called
+    when data/raw/YYYY-MM-DD does not exist, and access() returns False
+    for a directory that does not exist, so the copy is always skipped.
     """
 
     # Validate date is well formatted
@@ -74,26 +78,26 @@ def get_raw_data(date):
 
     raw_data_dir = path.join(_project_dir,'data','raw', date)
 
-    # Ensure directory exists and is writable
+    # Only copy if the directory exists and is writable (see known issue above)
     if access(raw_data_dir, W_OK):
         # Create directory, ignore if it exists
         makedirs(raw_data_dir, exist_ok=True)
 
-        # The next part seems to be prepared to retrieve data from a location
-        # and store it in the data/raw/YYYY-MM-DD directory for processing.
+        # Build one copy command per hour of the day:
+        # <magic><date>/conn.<hour>* data/raw/YYYY-MM-DD
         commands = [
-            shlex.split(_config['magic'] + f'{date}/conn.{x:02}* ' + raw_data_dir) 
+            shlex.split(_config['magic'] + f'{date}/conn.{x:02}* ' + raw_data_dir)
             for x in range(0,24)
         ]
 
-        # Attempting to run the previous commands in parallel
+        # Run the copy commands in parallel
         Parallel(n_jobs=24, backend='threading')(delayed(subprocess.run)(cmd) for cmd in commands)
 
 
 def remove_raw_data(date, force=False):
     """
-    Remove (delete) the content of the raw data directory 
-    for a given date.
+    Delete the raw data directory of a given date. Only runs if
+    remove_raw_data is set to true in the .env file, or if force=True.
     """
     # Validate date is well formatted
     dt.datetime.strptime(date, '%Y-%m-%d')