From 28d7bdb9225d2ceea9cbee84ba668fcf4f783fa7 Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Sun, 3 Nov 2024 11:43:44 +0100 Subject: [PATCH 01/34] Add instruction to run natively (#64) --- README.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 89941ac..18f3a3e 100644 --- a/README.md +++ b/README.md @@ -38,11 +38,13 @@ Each AIP model generates its own blocklist based on a specific criteria. The mai - A more experimental approach to increase blocklist efficiency. -## AIP Docker +## Installation -The best way to run AIP right now is using [Docker](etc/docker/README.md). +AIP can be run through Docker (recommended) or natively. In any case, make sure the data is correctly accessible in the `data/raw/` folder and that the files `data/external/` are correctly configured. + +### AIP Docker -## Usage +The best way to run AIP right now is using [Docker](etc/docker/README.md). AIP will automatically attempt to run all the models using the available data. Assuming the Zeek data is located in its usual location: @@ -57,6 +59,25 @@ To run AIP for a specific day: :~$ docker run --rm -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro -v ${PWD}/data/:/home/aip/AIP/data/:rw --name aip stratosphereips/aip:latest bin/aip YYYY-MM-DD ``` +### Local Installation + +To run AIP locally, follow the next steps: + +* Clone the AIP repository: + * `git clone https://github.com/stratosphereips/AIP.git ~/AIP` +* Access the AIP folder: + * `cd ~/AIP` +* Create a Python Virtual Environment to install the requirements + * `python3 -m venv aip-venv` +* Activate the virtual environment + * `source aip-venv/bin/activate` +* Install the Python requirements: + * `pip install -r requirements.txt` +* Set the PYTHONPATH environment variable so Python knows where to find all AIP components: + * `export PYTHONPATH=$(pwd)/lib` +* Let's test that AIP works with a simple test: + * `python3 bin/aip --help` + ## License The Stratosphere AIP tool is licensed under [GNU General Public License v3.0](https://github.com/stratosphereips/AIP/blob/main/LICENSE). From bc011160b7cf7853d7488f5e21f0f6cd98f3294f Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Sun, 3 Nov 2024 12:43:45 +0100 Subject: [PATCH 02/34] Add validation future date (#65) * Exit on invalid date * Raise exception on future date * Add test for checking future date raise exception --- bin/aip | 7 ++++++- lib/aip/utils/date_utils.py | 14 ++++++++------ tests/test_lib_aip_utils_date_utils.py | 9 +++++++++ 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/bin/aip b/bin/aip index d28055b..d68d548 100755 --- a/bin/aip +++ b/bin/aip @@ -31,6 +31,7 @@ __version__ = "1.0.0" import argparse import logging +import sys from datetime import date from os import makedirs from os import path @@ -74,7 +75,11 @@ def main(): logging.basicConfig(level=args.log_level, format=log_fmt) # Validate input date - run_date_day = validate_and_convert_date(args.date) + try: + run_date_day = validate_and_convert_date(args.date) + except ValueError as err: + logging.error(err) + sys.exit() # Run Alpha Model if args.model in ['Alpha', 'all']: diff --git a/lib/aip/utils/date_utils.py b/lib/aip/utils/date_utils.py index 6cdf829..a5f7936 100644 --- a/lib/aip/utils/date_utils.py +++ b/lib/aip/utils/date_utils.py @@ -1,5 +1,5 @@ -import logging from datetime import datetime +from datetime import date def validate_and_convert_date(date_str): @@ -7,8 +7,10 @@ def validate_and_convert_date(date_str): Validates a date string in 'YYYY-MM-DD' format and converts it to a date object. """ try: - dateobj = datetime.strptime(date_str, '%Y-%m-%d') - return dateobj.date() - except ValueError as e: - logging.error(f"Invalid date format for '{date_str}', expected YYYY-MM-DD") - raise ValueError(f"Invalid date format: {date_str}, expected YYYY-MM-DD") from e \ No newline at end of file + dateobj = datetime.strptime(date_str, '%Y-%m-%d').date() + date_today = date.today() + if dateobj > date_today: + raise ValueError(f"The input date '{date_str}' cannot be in the future.") + return dateobj + except ValueError as err: + raise ValueError(f"Invalid date format: {date_str}, expected YYYY-MM-DD") from err \ No newline at end of file diff --git a/tests/test_lib_aip_utils_date_utils.py b/tests/test_lib_aip_utils_date_utils.py index 79a663c..35e8319 100644 --- a/tests/test_lib_aip_utils_date_utils.py +++ b/tests/test_lib_aip_utils_date_utils.py @@ -1,6 +1,7 @@ import unittest import logging from datetime import date +from datetime import timedelta from lib.aip.utils.date_utils import validate_and_convert_date @@ -48,5 +49,13 @@ def test_edge_case(self): # Test leap years self.assertEqual(validate_and_convert_date("2024-02-29"), date(2024, 2, 29)) + def test_future_date(self): + # Test future date raises exception + future_date = (date.today() + timedelta(days=1)).isoformat() + + with self.assertRaises(ValueError): + validate_and_convert_date(future_date) + + if __name__ == '__main__': unittest.main() From 423b99d9e0d7bad347718880c474b8b160e254a1 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:04:28 +0000 Subject: [PATCH 03/34] Add kb file to host the kb class --- lib/aip/utils/knowledge_base.py | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 lib/aip/utils/knowledge_base.py diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py new file mode 100644 index 0000000..98687d0 --- /dev/null +++ b/lib/aip/utils/knowledge_base.py @@ -0,0 +1,4 @@ +""" +This module provides functionality for managing the +knowledge base used by AIP models. +""" From 31cc18ff43c85bb9236b93cf9dcf41dfd6ba0707 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:04:52 +0000 Subject: [PATCH 04/34] Move KB class and func. to own file --- lib/aip/models/prioritize.py | 107 +----------------------------- lib/aip/utils/knowledge_base.py | 113 ++++++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 105 deletions(-) diff --git a/lib/aip/models/prioritize.py b/lib/aip/models/prioritize.py index e666a22..9c2df61 100644 --- a/lib/aip/models/prioritize.py +++ b/lib/aip/models/prioritize.py @@ -27,119 +27,16 @@ import numpy as np import pandas as pd -import time from aip.data.access import data_path, get_attacks from aip.models.base import BaseModel from aip.utils.autoload import register, models -from datetime import date, datetime, timedelta +from aip.utils.knowledge_base import Knowledgebase +from datetime import date, timedelta from os import path from sklearn.ensemble import RandomForestClassifier -def _add_knowledge(last_knowledge, day): - print(f'DEBUG: PROCESSING DATE {day}') - st_time = time.time() - day = datetime.strptime(day, '%Y-%m-%d').date() - p = path.join(data_path, 'processed', 'prioritizers') - attacks = get_attacks(dates=[day]) - df = attacks[0] - if df.empty is True: - last_knowledge.to_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz'), - columns=['orig', 'flows', 'duration','bytes', 'packets', - 'mean_flows', 'mean_duration', 'mean_bytes', 'mean_packets', - 'days_active', 'first_seen', 'last_seen'], - index=False, compression='gzip') - return last_knowledge - df = df.rename(columns={"count": "flows"}) - df.loc[:, 'first_seen'] = day - df.loc[:, 'last_seen'] = day - df.loc[:, 'days_active'] = 1 - df.loc[df['flows'] == 0, 'flows'] = 1 - last_knowledge = pd.concat([last_knowledge, df]) - dates_min = last_knowledge.groupby('orig').first_seen.min() - dates_max = last_knowledge.groupby('orig').last_seen.max() - knowledge = last_knowledge.groupby('orig').sum(numeric_only=True) - knowledge.loc[:,'first_seen'] = dates_min - knowledge.loc[:,'last_seen'] = dates_max - knowledge.loc[:,'mean_flows'] = knowledge['flows']/knowledge['days_active'] - knowledge.loc[:,'mean_duration'] = knowledge['duration']/knowledge['flows'] - knowledge.loc[:,'mean_bytes'] = knowledge['bytes']/knowledge['flows'] - knowledge.loc[:,'mean_packets'] = knowledge['packets']/knowledge['flows'] - knowledge.reset_index(inplace=True) - knowledge.to_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz'), - columns=['orig', 'flows', 'duration','bytes', 'packets', - 'mean_flows', 'mean_duration', 'mean_bytes', 'mean_packets', - 'days_active', 'first_seen', 'last_seen'], - index=False, compression='gzip') - print(f'DEBUG: PROCESSED IN {(time.time() - st_time)/60} MINUTES.') - return knowledge - -def _build_knowledge(start, end): - dates = pd.date_range(start=start, end=end) - last_knowledge = pd.DataFrame() - for day in dates: - last_knowledge = _add_knowledge(last_knowledge, str(day.date())) - return last_knowledge - - -class Knowledgebase(): - ''' - Object that represents the IP features database - ''' - - def _check_date_param(self, day_unchk): - if day_unchk == 'yesterday': - day = str(date.today() - timedelta(days=1)) - elif type(day_unchk) is str: - # force is a date or throw and exception - day = str(datetime.strptime(day_unchk, '%Y-%m-%d').date()) - elif type(day_unchk) is date: - day = str(day_unchk) - return day - - def _load_knowledge_until(self, day): - day = self._check_date_param(day) - self.path = path.join(data_path, - 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') - if not path.exists(self.path): - self.build(end=datetime.strptime(day, '%Y-%m-%d').date()) - self.knowledge = pd.read_csv(self.path) - self.knowledge.loc[:, 'last_seen'] = pd.to_datetime(self.knowledge.last_seen) - self.knowledge.loc[:, 'first_seen'] = pd.to_datetime(self.knowledge.first_seen) - self.timeframe = (self.knowledge.last_seen.min(), self.knowledge.last_seen.max()) - - def __init__(self, load_until='yesterday'): - day = self._check_date_param(load_until) - self.path = path.join(data_path, - 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') - self._load_knowledge_until(day) - - - def build(self, start=date.today() - timedelta(days=2), end=date.today() - timedelta(days=1), force=False): - if path.exists(self.path) and not force: - print('Knowledge exists already. Use force=True to rebuild it') - return - # check if the snapshot for the start date exists - # if not, all the snapshots must be created again - p = path.join(data_path, 'processed', 'prioritizers') - if not path.exists(path.join(p, f'knowledgebase-{str(start)}-snapshot.gz')): - last_knowledge = _build_knowledge(start=start, end=end) - else: - days_ago = 1 - day = str(end - timedelta(days=days_ago)) - while not path.exists(path.join(p, f'knowledgebase-{day}-snapshot.gz')): - days_ago += 1 - day = str(end - timedelta(days=days_ago)) - last_knowledge = pd.read_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz')) - last_knowledge.loc[:, 'first_seen'] = pd.to_datetime(last_knowledge.first_seen).dt.date - last_knowledge.loc[:, 'last_seen'] = pd.to_datetime(last_knowledge.last_seen).dt.date - while days_ago >= 1: - days_ago -= 1 - day = str(end - timedelta(days=days_ago)) - last_knowledge = _add_knowledge(last_knowledge, day) - - @register class Consistent(BaseModel): ''' diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 98687d0..1532721 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -2,3 +2,116 @@ This module provides functionality for managing the knowledge base used by AIP models. """ + +import time +import pandas as pd +from datetime import date +from datetime import datetime +from datetime import timedelta +from os import path +from aip.data.access import data_path +from aip.data.access import get_attacks + + +def _add_knowledge(last_knowledge, day): + print(f'DEBUG: PROCESSING DATE {day}') + st_time = time.time() + day = datetime.strptime(day, '%Y-%m-%d').date() + p = path.join(data_path, 'processed', 'prioritizers') + attacks = get_attacks(dates=[day]) + df = attacks[0] + if df.empty is True: + last_knowledge.to_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz'), + columns=['orig', 'flows', 'duration','bytes', 'packets', + 'mean_flows', 'mean_duration', 'mean_bytes', 'mean_packets', + 'days_active', 'first_seen', 'last_seen'], + index=False, compression='gzip') + return last_knowledge + df = df.rename(columns={"count": "flows"}) + df.loc[:, 'first_seen'] = day + df.loc[:, 'last_seen'] = day + df.loc[:, 'days_active'] = 1 + df.loc[df['flows'] == 0, 'flows'] = 1 + last_knowledge = pd.concat([last_knowledge, df]) + dates_min = last_knowledge.groupby('orig').first_seen.min() + dates_max = last_knowledge.groupby('orig').last_seen.max() + knowledge = last_knowledge.groupby('orig').sum(numeric_only=True) + knowledge.loc[:,'first_seen'] = dates_min + knowledge.loc[:,'last_seen'] = dates_max + knowledge.loc[:,'mean_flows'] = knowledge['flows']/knowledge['days_active'] + knowledge.loc[:,'mean_duration'] = knowledge['duration']/knowledge['flows'] + knowledge.loc[:,'mean_bytes'] = knowledge['bytes']/knowledge['flows'] + knowledge.loc[:,'mean_packets'] = knowledge['packets']/knowledge['flows'] + knowledge.reset_index(inplace=True) + knowledge.to_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz'), + columns=['orig', 'flows', 'duration','bytes', 'packets', + 'mean_flows', 'mean_duration', 'mean_bytes', 'mean_packets', + 'days_active', 'first_seen', 'last_seen'], + index=False, compression='gzip') + print(f'DEBUG: PROCESSED IN {(time.time() - st_time)/60} MINUTES.') + return knowledge + + +def _build_knowledge(start, end): + dates = pd.date_range(start=start, end=end) + last_knowledge = pd.DataFrame() + for day in dates: + last_knowledge = _add_knowledge(last_knowledge, str(day.date())) + return last_knowledge + + +class Knowledgebase(): + ''' + Object that represents the IP features database + ''' + + def _check_date_param(self, day_unchk): + if day_unchk == 'yesterday': + day = str(date.today() - timedelta(days=1)) + elif type(day_unchk) is str: + # force is a date or throw and exception + day = str(datetime.strptime(day_unchk, '%Y-%m-%d').date()) + elif type(day_unchk) is date: + day = str(day_unchk) + return day + + def _load_knowledge_until(self, day): + day = self._check_date_param(day) + self.path = path.join(data_path, + 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') + if not path.exists(self.path): + self.build(end=datetime.strptime(day, '%Y-%m-%d').date()) + self.knowledge = pd.read_csv(self.path) + self.knowledge.loc[:, 'last_seen'] = pd.to_datetime(self.knowledge.last_seen) + self.knowledge.loc[:, 'first_seen'] = pd.to_datetime(self.knowledge.first_seen) + self.timeframe = (self.knowledge.last_seen.min(), self.knowledge.last_seen.max()) + + def __init__(self, load_until='yesterday'): + day = self._check_date_param(load_until) + self.path = path.join(data_path, + 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') + self._load_knowledge_until(day) + + + def build(self, start=date.today() - timedelta(days=2), end=date.today() - timedelta(days=1), force=False): + if path.exists(self.path) and not force: + print('Knowledge exists already. Use force=True to rebuild it') + return + # check if the snapshot for the start date exists + # if not, all the snapshots must be created again + p = path.join(data_path, 'processed', 'prioritizers') + if not path.exists(path.join(p, f'knowledgebase-{str(start)}-snapshot.gz')): + last_knowledge = _build_knowledge(start=start, end=end) + else: + days_ago = 1 + day = str(end - timedelta(days=days_ago)) + while not path.exists(path.join(p, f'knowledgebase-{day}-snapshot.gz')): + days_ago += 1 + day = str(end - timedelta(days=days_ago)) + last_knowledge = pd.read_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz')) + last_knowledge.loc[:, 'first_seen'] = pd.to_datetime(last_knowledge.first_seen).dt.date + last_knowledge.loc[:, 'last_seen'] = pd.to_datetime(last_knowledge.last_seen).dt.date + while days_ago >= 1: + days_ago -= 1 + day = str(end - timedelta(days=days_ago)) + last_knowledge = _add_knowledge(last_knowledge, day) \ No newline at end of file From 24604e247a57e6489abd03d8e3fb0a3b54df20a5 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:41:40 +0000 Subject: [PATCH 05/34] Move PC to own model file --- lib/aip/models/prioritize_consistent.py | 81 +++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 lib/aip/models/prioritize_consistent.py diff --git a/lib/aip/models/prioritize_consistent.py b/lib/aip/models/prioritize_consistent.py new file mode 100644 index 0000000..cfa6d1b --- /dev/null +++ b/lib/aip/models/prioritize_consistent.py @@ -0,0 +1,81 @@ +""" +AIP - Prioritize New/Consistent Algorithm + +This module prioritize the new/consistent attackers. + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +__authors__ = ["Joaquin Bogado "] +__contact__ = "stratosphere@aic.fel.cvut.cz" +__copyright__ = "Copyright 2022, Stratosphere Laboratory." +__credits__ = ["Joaquín Bogado"] +__deprecated__ = False +__license__ = "GPLv3" +__maintainer__ = "Joaquin Bogado" +__version__ = "0.0.1" + + +import numpy as np +import pandas as pd + +from aip.models.base import BaseModel +from aip.utils.autoload import register +from aip.utils.knowledge_base import Knowledgebase +from datetime import date, timedelta + + +@register +class Consistent(BaseModel): + ''' + Prioritize Consistent algorithm + ''' + + def __init__(self): + super().__init__() + # Weights from Thomas' Thesis + # nflows, conns, nbytes, npackets, + # mean_flows, mean_conns, mean_bytes, mean_packets + self.weights = [0.05, 0.05, 0.05, 0.05, 0.2, 0.2, 0.2, 0.2] + self.score_threshold = 0.00002 + self.min_ip_number = 5000 + + def _get_IP_scores(self): + days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) + features = self.db.knowledge[ + ['flows', 'duration','bytes', 'packets', 'mean_flows', + 'mean_duration', 'mean_bytes', 'mean_packets'] + ].values + normalized_features = np.zeros_like(features) + for i in range(8): + normalized_features[:,i] = (features[:,i] - features[:,i].min()) / (features[:,i].max() - features[:,i].min()) + ipscores = normalized_features * self.weights + ipscores = ipscores.sum(axis=1) + days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) + total_attack_time = np.array([x.days for x in (self.db.knowledge.last_seen - self.db.knowledge.first_seen)]) + aging = 1 - (days_since_last_seen/(days_since_last_seen + total_attack_time)) + ipscores *= aging + return ipscores + + def run(self, for_date=date.today()): + self.db = Knowledgebase(load_until=for_date - timedelta(days=1)) + ipscores = self._get_IP_scores() + df = pd.DataFrame() + df['ip'] = self.db.knowledge['orig'].values + df['score'] = ipscores + df = df.sort_values(by='score', ascending=False) + df = self.sanitize(df) + if len(df[df.score > self.score_threshold]) < self.min_ip_number: + df = df.iloc[:self.min_ip_number] + else: + df = df[df.score > self.score_threshold] + return df From 51cadb900dd96e8ad90fb0c6a640ed7c6c394d77 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:41:57 +0000 Subject: [PATCH 06/34] Move PN to own model file --- lib/aip/models/prioritize_new.py | 64 ++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 lib/aip/models/prioritize_new.py diff --git a/lib/aip/models/prioritize_new.py b/lib/aip/models/prioritize_new.py new file mode 100644 index 0000000..f31be18 --- /dev/null +++ b/lib/aip/models/prioritize_new.py @@ -0,0 +1,64 @@ +""" +AIP - Prioritize New/Consistent Algorithm + +This module prioritize the new/consistent attackers. + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +__authors__ = ["Joaquin Bogado "] +__contact__ = "stratosphere@aic.fel.cvut.cz" +__copyright__ = "Copyright 2022, Stratosphere Laboratory." +__credits__ = ["Joaquín Bogado"] +__deprecated__ = False +__license__ = "GPLv3" +__maintainer__ = "Joaquin Bogado" +__version__ = "0.0.1" + + +import numpy as np +import pandas as pd + +from aip.models.prioritize_consistent import Consistent +from aip.utils.autoload import register +from datetime import date + + +@register +class New(Consistent): + ''' + Prioritize New algorithm + ''' + + def __init__(self): + super().__init__() + # Weights from Thomas' Thesis + # nflows, conns, nbytes, npackets, + # mean_flows, mean_conns, mean_bytes, mean_packets + self.weights = [0.2, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.05] + self.score_threshold = 0.00001 + self.min_ip_number = 5000 + + def _get_IP_scores(self): + days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) + features = self.db.knowledge[ + ['flows', 'duration','bytes', 'packets', 'mean_flows', + 'mean_duration', 'mean_bytes', 'mean_packets'] + ].values + normalized_features = np.zeros_like(features) + for i in range(8): + normalized_features[:,i] = (features[:,i] - features[:,i].min()) / (features[:,i].max() - features[:,i].min()) + ipscores = normalized_features * self.weights + ipscores = ipscores.sum(axis=1) + aging = 2 / (2 + days_since_last_seen) + ipscores *= aging + return ipscores From 32987662b695979066c2a090429f1ee23670db5e Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:42:09 +0000 Subject: [PATCH 07/34] Move RF to own file --- lib/aip/models/random_forest.py | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 lib/aip/models/random_forest.py diff --git a/lib/aip/models/random_forest.py b/lib/aip/models/random_forest.py new file mode 100644 index 0000000..64e448a --- /dev/null +++ b/lib/aip/models/random_forest.py @@ -0,0 +1,85 @@ +""" +AIP - Prioritize New/Consistent Algorithm + +This module prioritize the new/consistent attackers. + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +__authors__ = ["Joaquin Bogado "] +__contact__ = "stratosphere@aic.fel.cvut.cz" +__copyright__ = "Copyright 2022, Stratosphere Laboratory." +__credits__ = ["Joaquín Bogado"] +__deprecated__ = False +__license__ = "GPLv3" +__maintainer__ = "Joaquin Bogado" +__version__ = "0.0.1" + + +import numpy as np +import pandas as pd + +from aip.data.access import get_attacks +from aip.models.base import BaseModel +from aip.utils.knowledge_base import Knowledgebase +from datetime import date +from datetime import timedelta +from sklearn.ensemble import RandomForestClassifier + + +class RandomForest(BaseModel): + ''' + Prioritize Random Forest from Thomas O'Hara's thesis + ''' + + def __init__(self): + super().__init__() + + def _get_training_target_set(self, for_date=date.today()): + + target = Knowledgebase(load_until=for_date - timedelta(days=1)) + target.attacks = get_attacks(dates=[(for_date - timedelta(days=1))])[-1] + days_since_last_seen = np.array([x.days for x in (pd.to_datetime(for_date) - target.knowledge.last_seen)]) + total_attack_time = np.array([x.days for x in (target.knowledge.last_seen - target.knowledge.first_seen)]) + target.knowledge['last_attack_days'] = days_since_last_seen + target.knowledge['first_attack_days'] = total_attack_time + target.features = target.knowledge[target.knowledge['orig'].isin(target.attacks['orig'])] + target.features = target.knowledge + target.X = target.features.drop(columns=['orig', 'first_seen', 'last_seen']).values + + training = Knowledgebase(load_until=for_date - timedelta(days=2)) + training.knowledge.loc[training.knowledge['flows'] == 0, 'flows'] = 1 + training.attacks = pd.concat(get_attacks(dates=[(for_date - timedelta(days=2))])) + days_since_last_seen = np.array([x.days for x in (pd.to_datetime(for_date) - training.knowledge.last_seen)]) + total_attack_time = np.array([x.days for x in (training.knowledge.last_seen - training.knowledge.first_seen)]) + training.knowledge['last_attack_days'] = days_since_last_seen + training.knowledge['first_attack_days'] = total_attack_time + training.knowledge['target'] = training.knowledge['orig'].isin(target.attacks['orig']).values.astype(int) + training.features = training.knowledge[training.knowledge['orig'].isin(training.attacks['orig'])] + training.features = training.knowledge + training.X = training.features.drop(columns=['orig', 'first_seen', 'last_seen', 'target']).values + training.Y = training.features['target'].values + return training, target + + def run(self, for_date=date.today()): + training, target = self._get_training_target_set(for_date=for_date) + clf = RandomForestClassifier(n_estimators=30) + if (target.X.shape[0] > 0) and (training.X.shape[0] > 0): + clf = clf.fit(training.X, training.Y) + pred = clf.predict(target.X) + df = target.features[pred.astype(bool)] + df = df.rename(columns={'orig': 'ip'}) + df = self.sanitize(df) + return df[['ip']] + else: + df = pd.DataFrame(columns=[['ip']]) + return df From a6af68048a9c194460b42e3eb05d0dc0c5411338 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:42:33 +0000 Subject: [PATCH 08/34] Update imports to new model files --- bin/aip | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/aip b/bin/aip index d68d548..f173c69 100755 --- a/bin/aip +++ b/bin/aip @@ -37,9 +37,9 @@ from os import makedirs from os import path from aip.data.access import data_path from aip.models.alpha import Alpha -from aip.models.prioritize import New -from aip.models.prioritize import Consistent -from aip.models.prioritize import RandomForest +from aip.models.prioritize_new import New +from aip.models.prioritize_consistent import Consistent +from aip.models.random_forest import RandomForest from aip.utils.date_utils import validate_and_convert_date From b7c15bb121f13b1ccb46ce3ed1d5a6e340fe4d6e Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:44:17 +0000 Subject: [PATCH 09/34] Update imports to match refactoring --- lib/aip/utils/generate_historical_blocklists.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/aip/utils/generate_historical_blocklists.py b/lib/aip/utils/generate_historical_blocklists.py index e56908a..af7fa7a 100755 --- a/lib/aip/utils/generate_historical_blocklists.py +++ b/lib/aip/utils/generate_historical_blocklists.py @@ -29,10 +29,9 @@ from aip.data.access import data_path, project_dir from aip.models.alpha import Alpha -from aip.models.prioritize import New -from aip.models.prioritize import Consistent -from aip.models.prioritize import RandomForest -from aip.models.prioritize import Knowledgebase +from lib.aip.models.prioritize_new import New +from lib.aip.models.prioritize_consistent import Consistent +from lib.aip.models.random_forest import RandomForest from aip.models.pareto import Pareto from datetime import date, timedelta from joblib import Parallel, delayed From f91b641141d299e5c44e1eba20ef0db0f0e1c05c Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:59:35 +0000 Subject: [PATCH 10/34] Move Alpha7 to own model file --- lib/aip/models/alpha.py | 6 ------ lib/aip/models/alpha7.py | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 lib/aip/models/alpha7.py diff --git a/lib/aip/models/alpha.py b/lib/aip/models/alpha.py index 5ae32b4..626372a 100644 --- a/lib/aip/models/alpha.py +++ b/lib/aip/models/alpha.py @@ -49,9 +49,3 @@ def run(self, for_date=date.today()): self.blocklist = attacks self.sanitize() return self.blocklist - -@register -class Alpha7(Alpha): - def __init__(self, lookback=7): - super().__init__() - self.lookback = lookback diff --git a/lib/aip/models/alpha7.py b/lib/aip/models/alpha7.py new file mode 100644 index 0000000..fc2b918 --- /dev/null +++ b/lib/aip/models/alpha7.py @@ -0,0 +1,34 @@ +""" +AIP - AlphaX Model + +This model filter the attackers of the previous X days. + +This program is free software: you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation, either version 3 of the License, or (at your option) any later +version. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. +You should have received a copy of the GNU General Public License along with +this program. If not, see . +""" + +__authors__ = ["Joaquin Bogado "] +__contact__ = "stratosphere@aic.fel.cvut.cz" +__copyright__ = "Copyright 2022, Stratosphere Laboratory." +__credits__ = ["Joaquín Bogado"] +__deprecated__ = False +__license__ = "GPLv3" +__maintainer__ = "Joaquin Bogado" +__version__ = "0.0.1" + +from aip.models.alpha import Alpha +from aip.utils.autoload import register + +@register +class Alpha7(Alpha): + def __init__(self, lookback=7): + super().__init__() + self.lookback = lookback From 4979fa79ae7149f2cf94b88df72607d028140d98 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 14:59:47 +0000 Subject: [PATCH 11/34] Update alpha7 import --- bin/aip | 1 + lib/aip/utils/generate_historical_blocklists.py | 1 + 2 files changed, 2 insertions(+) diff --git a/bin/aip b/bin/aip index f173c69..b2d1867 100755 --- a/bin/aip +++ b/bin/aip @@ -37,6 +37,7 @@ from os import makedirs from os import path from aip.data.access import data_path from aip.models.alpha import Alpha +from aip.models.alpha7 import Alpha7 from aip.models.prioritize_new import New from aip.models.prioritize_consistent import Consistent from aip.models.random_forest import RandomForest diff --git a/lib/aip/utils/generate_historical_blocklists.py b/lib/aip/utils/generate_historical_blocklists.py index af7fa7a..6992bee 100755 --- a/lib/aip/utils/generate_historical_blocklists.py +++ b/lib/aip/utils/generate_historical_blocklists.py @@ -29,6 +29,7 @@ from aip.data.access import data_path, project_dir from aip.models.alpha import Alpha +from aip.models.alpha7 import Alpha7 from lib.aip.models.prioritize_new import New from lib.aip.models.prioritize_consistent import Consistent from lib.aip.models.random_forest import RandomForest From acac4048d1461727c4bf57fc65b973bfd891402f Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 15:21:29 +0000 Subject: [PATCH 12/34] Refactor model imports --- lib/aip/models/alpha.py | 13 +++++++++---- lib/aip/models/base.py | 5 +++-- lib/aip/models/prioritize.py | 13 ++++++++----- lib/aip/models/prioritize_consistent.py | 3 ++- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/aip/models/alpha.py b/lib/aip/models/alpha.py index 626372a..db19e79 100644 --- a/lib/aip/models/alpha.py +++ b/lib/aip/models/alpha.py @@ -27,11 +27,11 @@ import pandas as pd from aip.models.base import BaseModel -from aip.utils.autoload import register, models +from aip.utils.autoload import register from aip.data.access import get_attacks -from datetime import date, timedelta +from datetime import date +from datetime import timedelta -#__models__ = [] @register class Alpha(BaseModel): @@ -42,10 +42,15 @@ def __init__(self, lookback=1): def run(self, for_date=date.today()): start = str(for_date - timedelta(days=self.lookback)) end = str(for_date - timedelta(days=1)) - # get all the attackers IPs + + # Get all the attackers IPs attacks = get_attacks(start, end, usecols=['orig']) attacks = pd.concat(attacks).drop_duplicates() attacks = attacks.rename(columns={'orig':'ip'}) + self.blocklist = attacks + + # Remove IPs from do_not_block_these_ips.csv self.sanitize() + return self.blocklist diff --git a/lib/aip/models/base.py b/lib/aip/models/base.py index 4539cac..40c884a 100644 --- a/lib/aip/models/base.py +++ b/lib/aip/models/base.py @@ -26,10 +26,11 @@ import pandas as pd import logging +from os import path from aip.data.access import data_dir -from aip.utils.autoload import register, models -from os import path +from aip.utils.autoload import models +from aip.utils.autoload import register # @register class BaseModel(): diff --git a/lib/aip/models/prioritize.py b/lib/aip/models/prioritize.py index 9c2df61..ea18d8a 100644 --- a/lib/aip/models/prioritize.py +++ b/lib/aip/models/prioritize.py @@ -27,14 +27,17 @@ import numpy as np import pandas as pd +from datetime import date +from datetime import timedelta +from os import path +from sklearn.ensemble import RandomForestClassifier -from aip.data.access import data_path, get_attacks +from aip.data.access import data_path +from aip.data.access import get_attacks from aip.models.base import BaseModel -from aip.utils.autoload import register, models +from aip.utils.autoload import models +from aip.utils.autoload import register from aip.utils.knowledge_base import Knowledgebase -from datetime import date, timedelta -from os import path -from sklearn.ensemble import RandomForestClassifier @register diff --git a/lib/aip/models/prioritize_consistent.py b/lib/aip/models/prioritize_consistent.py index cfa6d1b..b465aa6 100644 --- a/lib/aip/models/prioritize_consistent.py +++ b/lib/aip/models/prioritize_consistent.py @@ -31,7 +31,8 @@ from aip.models.base import BaseModel from aip.utils.autoload import register from aip.utils.knowledge_base import Knowledgebase -from datetime import date, timedelta +from datetime import date +from datetime import timedelta @register From 9e2bcf464cde7ba0f310b40b7048f9d124dbe4f3 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 20:26:47 +0000 Subject: [PATCH 13/34] remove file --- lib/aip/models/prioritize.py | 166 ----------------------------------- 1 file changed, 166 deletions(-) delete mode 100644 lib/aip/models/prioritize.py diff --git a/lib/aip/models/prioritize.py b/lib/aip/models/prioritize.py deleted file mode 100644 index ea18d8a..0000000 --- a/lib/aip/models/prioritize.py +++ /dev/null @@ -1,166 +0,0 @@ -""" -AIP - Prioritize New/Consistent Algorithm - -This module prioritize the new/consistent attackers. - -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program. If not, see . -""" - -__authors__ = ["Joaquin Bogado "] -__contact__ = "stratosphere@aic.fel.cvut.cz" -__copyright__ = "Copyright 2022, Stratosphere Laboratory." -__credits__ = ["Joaquín Bogado"] -__deprecated__ = False -__license__ = "GPLv3" -__maintainer__ = "Joaquin Bogado" -__version__ = "0.0.1" - - -import numpy as np -import pandas as pd -from datetime import date -from datetime import timedelta -from os import path -from sklearn.ensemble import RandomForestClassifier - -from aip.data.access import data_path -from aip.data.access import get_attacks -from aip.models.base import BaseModel -from aip.utils.autoload import models -from aip.utils.autoload import register -from aip.utils.knowledge_base import Knowledgebase - - -@register -class Consistent(BaseModel): - ''' - Prioritize Consistent algorithm - ''' - - def __init__(self): - super().__init__() - # Weights from Thomas' Thesis - # nflows, conns, nbytes, npackets, - # mean_flows, mean_conns, mean_bytes, mean_packets - self.weights = [0.05, 0.05, 0.05, 0.05, 0.2, 0.2, 0.2, 0.2] - self.score_threshold = 0.00002 - self.min_ip_number = 5000 - - def _get_IP_scores(self): - days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) - features = self.db.knowledge[ - ['flows', 'duration','bytes', 'packets', 'mean_flows', - 'mean_duration', 'mean_bytes', 'mean_packets'] - ].values - normalized_features = np.zeros_like(features) - for i in range(8): - normalized_features[:,i] = (features[:,i] - features[:,i].min()) / (features[:,i].max() - features[:,i].min()) - ipscores = normalized_features * self.weights - ipscores = ipscores.sum(axis=1) - days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) - total_attack_time = np.array([x.days for x in (self.db.knowledge.last_seen - self.db.knowledge.first_seen)]) - aging = 1 - (days_since_last_seen/(days_since_last_seen + total_attack_time)) - ipscores *= aging - return ipscores - - def run(self, for_date=date.today()): - self.db = Knowledgebase(load_until=for_date - timedelta(days=1)) - ipscores = self._get_IP_scores() - df = pd.DataFrame() - df['ip'] = self.db.knowledge['orig'].values - df['score'] = ipscores - df = df.sort_values(by='score', ascending=False) - df = self.sanitize(df) - if len(df[df.score > self.score_threshold]) < self.min_ip_number: - df = df.iloc[:self.min_ip_number] - else: - df = df[df.score > self.score_threshold] - return df - - -@register -class New(Consistent): - ''' - Prioritize New algorithm - ''' - - def __init__(self): - super().__init__() - # Weights from Thomas' Thesis - # nflows, conns, nbytes, npackets, - # mean_flows, mean_conns, mean_bytes, mean_packets - self.weights = [0.2, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.05] - self.score_threshold = 0.00001 - self.min_ip_number = 5000 - - def _get_IP_scores(self): - days_since_last_seen = np.array([x.days for x in (pd.to_datetime(date.today()) - self.db.knowledge.last_seen)]) - features = self.db.knowledge[ - ['flows', 'duration','bytes', 'packets', 'mean_flows', - 'mean_duration', 'mean_bytes', 'mean_packets'] - ].values - normalized_features = np.zeros_like(features) - for i in range(8): - normalized_features[:,i] = (features[:,i] - features[:,i].min()) / (features[:,i].max() - features[:,i].min()) - ipscores = normalized_features * self.weights - ipscores = ipscores.sum(axis=1) - aging = 2 / (2 + days_since_last_seen) - ipscores *= aging - return ipscores - -class RandomForest(BaseModel): - ''' - Prioritize Random Forest from Thomas O'Hara's thesis - ''' - - def __init__(self): - super().__init__() - - def _get_training_target_set(self, for_date=date.today()): - - target = Knowledgebase(load_until=for_date - timedelta(days=1)) - target.attacks = get_attacks(dates=[(for_date - timedelta(days=1))])[-1] - days_since_last_seen = np.array([x.days for x in (pd.to_datetime(for_date) - target.knowledge.last_seen)]) - total_attack_time = np.array([x.days for x in (target.knowledge.last_seen - target.knowledge.first_seen)]) - target.knowledge['last_attack_days'] = days_since_last_seen - target.knowledge['first_attack_days'] = total_attack_time - target.features = target.knowledge[target.knowledge['orig'].isin(target.attacks['orig'])] - target.features = target.knowledge - target.X = target.features.drop(columns=['orig', 'first_seen', 'last_seen']).values - - training = Knowledgebase(load_until=for_date - timedelta(days=2)) - training.knowledge.loc[training.knowledge['flows'] == 0, 'flows'] = 1 - training.attacks = pd.concat(get_attacks(dates=[(for_date - timedelta(days=2))])) - days_since_last_seen = np.array([x.days for x in (pd.to_datetime(for_date) - training.knowledge.last_seen)]) - total_attack_time = np.array([x.days for x in (training.knowledge.last_seen - training.knowledge.first_seen)]) - training.knowledge['last_attack_days'] = days_since_last_seen - training.knowledge['first_attack_days'] = total_attack_time - training.knowledge['target'] = training.knowledge['orig'].isin(target.attacks['orig']).values.astype(int) - training.features = training.knowledge[training.knowledge['orig'].isin(training.attacks['orig'])] - training.features = training.knowledge - training.X = training.features.drop(columns=['orig', 'first_seen', 'last_seen', 'target']).values - training.Y = training.features['target'].values - return training, target - - def run(self, for_date=date.today()): - training, target = self._get_training_target_set(for_date=for_date) - clf = RandomForestClassifier(n_estimators=30) - if (target.X.shape[0] > 0) and (training.X.shape[0] > 0): - clf = clf.fit(training.X, training.Y) - pred = clf.predict(target.X) - df = target.features[pred.astype(bool)] - df = df.rename(columns={'orig': 'ip'}) - df = self.sanitize(df) - return df[['ip']] - else: - df = pd.DataFrame(columns=[['ip']]) - return df From 2c6fa38f09f9726eedf432da232840f9c8f24f29 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:01:31 +0000 Subject: [PATCH 14/34] Add wrapper to rebuild KB since date --- lib/aip/utils/knowledge_base.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 1532721..ed930de 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -60,6 +60,23 @@ def _build_knowledge(start, end): return last_knowledge +def _rebuild(start_date, log_level=logging.ERROR): + """ + Wrapper to rebuild the knowledge base from the specified start date to today. + """ + # Configure logger + logger = logging.getLogger(__name__) + logging.basicConfig(level=log_level) + + st_time = time.time() + current_date = date.today() + logger.info(f"_rebuild - Rebuilding knowledge base from {start_date} to {current_date}") + + _build_knowledge(start=start_date, end=current_date, log_level=log_level) + + logger.info(f'_rebuild - Rebuilding knowledge base finished in {(time.time() - st_time)/60} minutes') + + class Knowledgebase(): ''' Object that represents the IP features database From 7a78304875a0a9d5b265b6793572adfc62627186 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:02:29 +0000 Subject: [PATCH 15/34] Add logging to _build_knowledge --- lib/aip/utils/knowledge_base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index ed930de..6d61e76 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -3,6 +3,7 @@ knowledge base used by AIP models. """ +import logging import time import pandas as pd from datetime import date @@ -52,11 +53,16 @@ def _add_knowledge(last_knowledge, day): return knowledge -def _build_knowledge(start, end): +def _build_knowledge(start, end, log_level=logging.ERROR): + # Configure logger + logger = logging.getLogger(__name__) + logging.basicConfig(level=log_level) + dates = pd.date_range(start=start, end=end) last_knowledge = pd.DataFrame() for day in dates: - last_knowledge = _add_knowledge(last_knowledge, str(day.date())) + logger.debug(f'_build_knowledge - Building knowledge base for {day}') + last_knowledge = _add_knowledge(last_knowledge, str(day.date()), log_level=log_level) return last_knowledge From ad1c79aab4139a517420d9a74ee4681a9193e516 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:02:57 +0000 Subject: [PATCH 16/34] Add logging to _add_knowledge --- lib/aip/utils/knowledge_base.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 6d61e76..d7b20d7 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -14,12 +14,19 @@ from aip.data.access import get_attacks -def _add_knowledge(last_knowledge, day): - print(f'DEBUG: PROCESSING DATE {day}') +def _add_knowledge(last_knowledge, day, log_level=logging.ERROR): + # Configure logger + logger = logging.getLogger(__name__) + logging.basicConfig(level=log_level) + + logger.debug(f'_add_knowledge - Processing date {day}') st_time = time.time() day = datetime.strptime(day, '%Y-%m-%d').date() + p = path.join(data_path, 'processed', 'prioritizers') + attacks = get_attacks(dates=[day]) + df = attacks[0] if df.empty is True: last_knowledge.to_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz'), @@ -33,6 +40,7 @@ def _add_knowledge(last_knowledge, day): df.loc[:, 'last_seen'] = day df.loc[:, 'days_active'] = 1 df.loc[df['flows'] == 0, 'flows'] = 1 + last_knowledge = pd.concat([last_knowledge, df]) dates_min = last_knowledge.groupby('orig').first_seen.min() dates_max = last_knowledge.groupby('orig').last_seen.max() @@ -49,7 +57,8 @@ def _add_knowledge(last_knowledge, day): 'mean_flows', 'mean_duration', 'mean_bytes', 'mean_packets', 'days_active', 'first_seen', 'last_seen'], index=False, compression='gzip') - print(f'DEBUG: PROCESSED IN {(time.time() - st_time)/60} MINUTES.') + + logger.debug(f'_add_knowledge - Processed in {(time.time() - st_time)/60} minutes') return knowledge From 65e7f509f0b73f0e07e990f76b6bb4889884f67b Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:03:08 +0000 Subject: [PATCH 17/34] Add logging to KB class --- lib/aip/utils/knowledge_base.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index d7b20d7..5b0d757 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -119,19 +119,24 @@ def _load_knowledge_until(self, day): self.timeframe = (self.knowledge.last_seen.min(), self.knowledge.last_seen.max()) def __init__(self, load_until='yesterday'): + # Set up the logger for the class + self.logger = logging.getLogger(self.__class__.__name__) + day = self._check_date_param(load_until) - self.path = path.join(data_path, - 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') + + self.path = path.join(data_path, 'processed', 'prioritizers', f'knowledgebase-{day}-snapshot.gz') self._load_knowledge_until(day) def build(self, start=date.today() - timedelta(days=2), end=date.today() - timedelta(days=1), force=False): if path.exists(self.path) and not force: - print('Knowledge exists already. Use force=True to rebuild it') + self.logger.error('The knowledge already exists. Use force=True to force its rebuild') return + # check if the snapshot for the start date exists # if not, all the snapshots must be created again p = path.join(data_path, 'processed', 'prioritizers') + if not path.exists(path.join(p, f'knowledgebase-{str(start)}-snapshot.gz')): last_knowledge = _build_knowledge(start=start, end=end) else: @@ -146,4 +151,4 @@ def build(self, start=date.today() - timedelta(days=2), end=date.today() - timed while days_ago >= 1: days_ago -= 1 day = str(end - timedelta(days=days_ago)) - last_knowledge = _add_knowledge(last_knowledge, day) \ No newline at end of file + last_knowledge = _add_knowledge(last_knowledge, day) From 9a06970f5ae7331d1750fefcea1111f62ba978b0 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:03:54 +0000 Subject: [PATCH 18/34] Add rebuild-kb argument to bin/aip --- bin/aip | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/aip b/bin/aip index b2d1867..a00138d 100755 --- a/bin/aip +++ b/bin/aip @@ -64,10 +64,11 @@ def run_model(aip_model_name, aip_model, date_day): def main(): parser = argparse.ArgumentParser(description='Attacker IP Prioritization (AIP) Tool') - parser.add_argument('--date', type=str, help='The date for running the models in YYYY-MM-DD format. Defaults to today.', default=str(date.today())) - parser.add_argument('--model', type=str, choices=['Alpha', 'Alpha7', 'Prioritize_New', 'Prioritize_Consistent', 'Random_Forest', 'all'], default='all', help='Select AIP model to run. Defaults to all.') parser.add_argument('-d', '--debug', required=False, help="Debugging mode.", action="store_const", dest="log_level", const=logging.DEBUG, default=logging.ERROR,) parser.add_argument('-v', '--verbose', required=False, help="Verbose mode", action="store_const", dest="log_level", const=logging.INFO,) + parser.add_argument('--date', type=str, help='The date for running the models in YYYY-MM-DD format. Defaults to today.', default=str(date.today())) + parser.add_argument('--model', type=str, choices=['Alpha', 'Alpha7', 'Prioritize_New', 'Prioritize_Consistent', 'Random_Forest', 'all'], default='all', help='Select AIP model to run. Defaults to all.') + parser.add_argument('--rebuild-kb', type=str, help='Rebuild knowledge base from a specific date (YYYY-MM-DD).') args = parser.parse_args() From f792d153a51fb6d8d81eda7d552c6177ffd446f6 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:06:09 +0000 Subject: [PATCH 19/34] Call _rebuild for kb rebuilt from date --- bin/aip | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/bin/aip b/bin/aip index a00138d..0f7795e 100755 --- a/bin/aip +++ b/bin/aip @@ -42,6 +42,7 @@ from aip.models.prioritize_new import New from aip.models.prioritize_consistent import Consistent from aip.models.random_forest import RandomForest from aip.utils.date_utils import validate_and_convert_date +from aip.utils.knowledge_base import _rebuild def run_model(aip_model_name, aip_model, date_day): @@ -73,14 +74,25 @@ def main(): args = parser.parse_args() # Set up logging + logger = logging.getLogger(__name__) log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(level=args.log_level, format=log_fmt) + if args.rebuild_kb: + try: + rebuild_date = validate_and_convert_date(args.rebuild_kb) + except ValueError as err: + logger.error(err) + sys.exit() + logger.info(f"Rebuilding knowledge base from {rebuild_date} onwards") + _rebuild(start_date=rebuild_date, log_level=args.log_level) + sys.exit() + # Validate input date try: run_date_day = validate_and_convert_date(args.date) except ValueError as err: - logging.error(err) + logger.error(err) sys.exit() # Run Alpha Model From a595688f3302c097525d086f6908d96843a7d971 Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:23:51 +0000 Subject: [PATCH 20/34] Catch non existing knowledgebase retrieval --- bin/aip | 4 ++-- lib/aip/utils/knowledge_base.py | 23 +++++++++++++---------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/bin/aip b/bin/aip index 0f7795e..8a2ba5a 100755 --- a/bin/aip +++ b/bin/aip @@ -59,8 +59,8 @@ def run_model(aip_model_name, aip_model, date_day): blocklist = aip_model.run(date_day) blocklist.to_csv(path.join(model_output_dir, f'AIP-{aip_model_name}-{str(date_day)}.csv.gz'), index=False, compression='gzip') logging.info(f"{aip_model_name} model completed successfully.") - except Exception as e: - logging.error(f"Error running {aip_model_name} model: {e}", exc_info=True) + except Exception as err: + logging.error(f"Error running {aip_model_name} model: {err}") def main(): diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 5b0d757..4f32c1c 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -141,14 +141,17 @@ def build(self, start=date.today() - timedelta(days=2), end=date.today() - timed last_knowledge = _build_knowledge(start=start, end=end) else: days_ago = 1 - day = str(end - timedelta(days=days_ago)) - while not path.exists(path.join(p, f'knowledgebase-{day}-snapshot.gz')): - days_ago += 1 + try: day = str(end - timedelta(days=days_ago)) - last_knowledge = pd.read_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz')) - last_knowledge.loc[:, 'first_seen'] = pd.to_datetime(last_knowledge.first_seen).dt.date - last_knowledge.loc[:, 'last_seen'] = pd.to_datetime(last_knowledge.last_seen).dt.date - while days_ago >= 1: - days_ago -= 1 - day = str(end - timedelta(days=days_ago)) - last_knowledge = _add_knowledge(last_knowledge, day) + while not path.exists(path.join(p, f'knowledgebase-{day}-snapshot.gz')): + days_ago += 1 + day = str(end - timedelta(days=days_ago)) + last_knowledge = pd.read_csv(path.join(p, f'knowledgebase-{day}-snapshot.gz')) + last_knowledge.loc[:, 'first_seen'] = pd.to_datetime(last_knowledge.first_seen).dt.date + last_knowledge.loc[:, 'last_seen'] = pd.to_datetime(last_knowledge.last_seen).dt.date + while days_ago >= 1: + days_ago -= 1 + day = str(end - timedelta(days=days_ago)) + last_knowledge = _add_knowledge(last_knowledge, day) + except OverflowError as err: + raise Exception(f"There is not Knowledge Base for the model to run (Required: {end})") From fae045976efa2141579c36e3fd0b3822fac1ef8a Mon Sep 17 00:00:00 2001 From: verovaleros Date: Sun, 3 Nov 2024 21:32:15 +0000 Subject: [PATCH 21/34] Pass on log level to model.run --- bin/aip | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/aip b/bin/aip index 8a2ba5a..016c0cb 100755 --- a/bin/aip +++ b/bin/aip @@ -45,7 +45,7 @@ from aip.utils.date_utils import validate_and_convert_date from aip.utils.knowledge_base import _rebuild -def run_model(aip_model_name, aip_model, date_day): +def run_model(aip_model_name, aip_model, date_day, log_level=logging.ERROR): """ Run a given model with exception handling """ @@ -95,25 +95,26 @@ def main(): logger.error(err) sys.exit() + logging.info(f"Running {args.model} model(s) for date {run_date_day}.") # Run Alpha Model if args.model in ['Alpha', 'all']: - run_model('Alpha', Alpha(), run_date_day) + run_model('Alpha', Alpha(), run_date_day, args.log_level) # Alpha 7 Model if args.model in ['Alpha7', 'all']: - run_model('Alpha7', Alpha(lookback=7), run_date_day) + run_model('Alpha7', Alpha(lookback=7), run_date_day, args.log_level) # Prioritize New Model if args.model in ['Prioritize_New', 'all']: - run_model('Prioritize_New', New(), run_date_day) + run_model('Prioritize_New', New(), run_date_day, args.log_level) # Prioritize Consistent Model if args.model in ['Prioritize_Consistent', 'all']: - run_model('Prioritize_Consistent', Consistent(), run_date_day) + run_model('Prioritize_Consistent', Consistent(), run_date_day, args.log_level) # Prioritize Random Forest Model if args.model in ['Random_Forest', 'all']: - run_model('Random_Forest', RandomForest(), run_date_day) + run_model('Random_Forest', RandomForest(), run_date_day, args.log_level) if __name__ == '__main__': main() From 482056a4963f68d97f83ede447a750f47828d84a Mon Sep 17 00:00:00 2001 From: verovaleros Date: Mon, 4 Nov 2024 17:20:35 +0000 Subject: [PATCH 22/34] Instaciate Alpha7 instead of Alpha(lookback=7) --- bin/aip | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/aip b/bin/aip index 016c0cb..0c746a7 100755 --- a/bin/aip +++ b/bin/aip @@ -102,7 +102,7 @@ def main(): # Alpha 7 Model if args.model in ['Alpha7', 'all']: - run_model('Alpha7', Alpha(lookback=7), run_date_day, args.log_level) + run_model('Alpha7', Alpha7(), run_date_day, args.log_level) # Prioritize New Model if args.model in ['Prioritize_New', 'all']: From f2297147dd74345ff6a391cea9a8cfd0b54eb34c Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Mon, 4 Nov 2024 19:07:19 +0100 Subject: [PATCH 23/34] Knowledge base rebuild take start and end dates via arguments (#72) * Catch ValueError once not every time * Rename st_time to start_time * _rebuild: add end_date argument to * Change logging level to debug --- bin/aip | 56 ++++++++++++++++----------------- lib/aip/utils/knowledge_base.py | 11 +++---- 2 files changed, 32 insertions(+), 35 deletions(-) diff --git a/bin/aip b/bin/aip index 0c746a7..f3879b9 100755 --- a/bin/aip +++ b/bin/aip @@ -78,43 +78,41 @@ def main(): log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(level=args.log_level, format=log_fmt) - if args.rebuild_kb: - try: + try: + # Rebuild knowledge base + if args.rebuild_kb: rebuild_date = validate_and_convert_date(args.rebuild_kb) - except ValueError as err: - logger.error(err) + logger.info(f"Rebuilding knowledge base from {rebuild_date} onwards") + _rebuild(start_date=rebuild_date, log_level=args.log_level) sys.exit() - logger.info(f"Rebuilding knowledge base from {rebuild_date} onwards") - _rebuild(start_date=rebuild_date, log_level=args.log_level) - sys.exit() - # Validate input date - try: - run_date_day = validate_and_convert_date(args.date) - except ValueError as err: - logger.error(err) - sys.exit() + # Validate input date + if args.date: + run_date_day = validate_and_convert_date(args.date) - logging.info(f"Running {args.model} model(s) for date {run_date_day}.") - # Run Alpha Model - if args.model in ['Alpha', 'all']: - run_model('Alpha', Alpha(), run_date_day, args.log_level) + logging.info(f"Running {args.model} model(s) for date {run_date_day}.") + # Run Alpha Model + if args.model in ['Alpha', 'all']: + run_model('Alpha', Alpha(), run_date_day, args.log_level) - # Alpha 7 Model - if args.model in ['Alpha7', 'all']: - run_model('Alpha7', Alpha7(), run_date_day, args.log_level) + # Alpha 7 Model + if args.model in ['Alpha7', 'all']: + run_model('Alpha7', Alpha7(), run_date_day, args.log_level) - # Prioritize New Model - if args.model in ['Prioritize_New', 'all']: - run_model('Prioritize_New', New(), run_date_day, args.log_level) + # Prioritize New Model + if args.model in ['Prioritize_New', 'all']: + run_model('Prioritize_New', New(), run_date_day, args.log_level) - # Prioritize Consistent Model - if args.model in ['Prioritize_Consistent', 'all']: - run_model('Prioritize_Consistent', Consistent(), run_date_day, args.log_level) + # Prioritize Consistent Model + if args.model in ['Prioritize_Consistent', 'all']: + run_model('Prioritize_Consistent', Consistent(), run_date_day, args.log_level) - # Prioritize Random Forest Model - if args.model in ['Random_Forest', 'all']: - run_model('Random_Forest', RandomForest(), run_date_day, args.log_level) + # Prioritize Random Forest Model + if args.model in ['Random_Forest', 'all']: + run_model('Random_Forest', RandomForest(), run_date_day, args.log_level) + except ValueError as err: + logger.error(err) + sys.exit() if __name__ == '__main__': main() diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 4f32c1c..8e970ab 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -75,7 +75,7 @@ def _build_knowledge(start, end, log_level=logging.ERROR): return last_knowledge -def _rebuild(start_date, log_level=logging.ERROR): +def _rebuild(start_date, end_date=date.today(), log_level=logging.ERROR): """ Wrapper to rebuild the knowledge base from the specified start date to today. """ @@ -83,13 +83,12 @@ def _rebuild(start_date, log_level=logging.ERROR): logger = logging.getLogger(__name__) logging.basicConfig(level=log_level) - st_time = time.time() - current_date = date.today() - logger.info(f"_rebuild - Rebuilding knowledge base from {start_date} to {current_date}") + start_time = time.time() + logger.debug(f"_rebuild - Rebuilding knowledge base from {start_date} to {end_date}") - _build_knowledge(start=start_date, end=current_date, log_level=log_level) + _build_knowledge(start=start_date, end=end_date, log_level=log_level) - logger.info(f'_rebuild - Rebuilding knowledge base finished in {(time.time() - st_time)/60} minutes') + logger.debug(f'_rebuild - Rebuilding knowledge base finished in {(time.time() - start_time)/60} minutes') class Knowledgebase(): From 55df89a640f3592317e4c702b5d7e1067adf775b Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Fri, 8 Nov 2024 10:38:29 +0100 Subject: [PATCH 24/34] Add scalable AIP arguments (#74) * Fix logger name to smth meaningful * Add subparser * Migrate to 'run-models' command * Migrate to 'rebuild-kb' command * Validate start_date once, not on every command * Set models default when not specified * Update to reflect change in variable names * Add some commands to the README --- README.md | 17 +++++++++++------ bin/aip | 51 +++++++++++++++++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 18f3a3e..d00bf2d 100644 --- a/README.md +++ b/README.md @@ -53,12 +53,6 @@ AIP will automatically attempt to run all the models using the available data. A :~$ docker run --rm -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro -v ${PWD}/data/:/home/aip/AIP/data/:rw --name aip stratosphereips/aip:latest bin/aip ``` -To run AIP for a specific day: -```bash -:~$ cd AIP -:~$ docker run --rm -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro -v ${PWD}/data/:/home/aip/AIP/data/:rw --name aip stratosphereips/aip:latest bin/aip YYYY-MM-DD -``` - ### Local Installation To run AIP locally, follow the next steps: @@ -78,6 +72,17 @@ To run AIP locally, follow the next steps: * Let's test that AIP works with a simple test: * `python3 bin/aip --help` +### AIP Commands + +Get all the full list of options with `bin/aip --help`: + +- Run AIP for a specific day: + - `bin/aip -v run-models --start-date YYYY-MM-DD` +- Run AIP for a specific day and model: + - `bin/aip -v run-models --start-date YYYY-MM-DD --model Alpha` +- Rebuild AIP knowledge base from a specific date: + - `bin/aip -d rebuild-kb --start-date YYYY-MM-DD` + ## License The Stratosphere AIP tool is licensed under [GNU General Public License v3.0](https://github.com/stratosphereips/AIP/blob/main/LICENSE). diff --git a/bin/aip b/bin/aip index f3879b9..c431564 100755 --- a/bin/aip +++ b/bin/aip @@ -67,49 +67,64 @@ def main(): parser = argparse.ArgumentParser(description='Attacker IP Prioritization (AIP) Tool') parser.add_argument('-d', '--debug', required=False, help="Debugging mode.", action="store_const", dest="log_level", const=logging.DEBUG, default=logging.ERROR,) parser.add_argument('-v', '--verbose', required=False, help="Verbose mode", action="store_const", dest="log_level", const=logging.INFO,) - parser.add_argument('--date', type=str, help='The date for running the models in YYYY-MM-DD format. Defaults to today.', default=str(date.today())) - parser.add_argument('--model', type=str, choices=['Alpha', 'Alpha7', 'Prioritize_New', 'Prioritize_Consistent', 'Random_Forest', 'all'], default='all', help='Select AIP model to run. Defaults to all.') - parser.add_argument('--rebuild-kb', type=str, help='Rebuild knowledge base from a specific date (YYYY-MM-DD).') + + subparsers = parser.add_subparsers(dest="command", help="commands") + + # Run Models command + run_models_parser = subparsers.add_parser("run-models", help="Run models with optional date range.") + run_models_parser.add_argument('--start-date', type=str, help='The start date for running the models (YYYY-MM-DD). Default is today.', default=str(date.today())) + run_models_parser.add_argument('--model', type=str, choices=['Alpha', 'Alpha7', 'Prioritize_New', 'Prioritize_Consistent', 'Random_Forest', 'all'], default='all', help='Select AIP model to run. Default is all models.') + + # Rebuild KB command + rebuild_kb_parser = subparsers.add_parser("rebuild-kb", help="Rebuild the knowledge base from a specific date.") + rebuild_kb_parser.add_argument("--start-date", required=True, type=str, help="The date from which to rebuild the knowledge base in YYYY-MM-DD format.") args = parser.parse_args() # Set up logging - logger = logging.getLogger(__name__) + logger = logging.getLogger('aip') log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(level=args.log_level, format=log_fmt) try: + # Validate start_date once not on every command + if hasattr(args,'start_date'): + aip_start_date = validate_and_convert_date(args.start_date) + else: + # If not set, default to today() + aip_start_date = date.today() + + # If model is not set, default to today + if not hasattr(args,'model'): + args.model = 'all' + # Rebuild knowledge base - if args.rebuild_kb: - rebuild_date = validate_and_convert_date(args.rebuild_kb) - logger.info(f"Rebuilding knowledge base from {rebuild_date} onwards") - _rebuild(start_date=rebuild_date, log_level=args.log_level) + if args.command == "rebuild-kb": + logger.info(f"Rebuilding knowledge base from {aip_start_date} onwards") + _rebuild(start_date=aip_start_date, log_level=args.log_level) sys.exit() - # Validate input date - if args.date: - run_date_day = validate_and_convert_date(args.date) - - logging.info(f"Running {args.model} model(s) for date {run_date_day}.") + logging.info(f"Running {args.model} model(s) for date {aip_start_date}.") # Run Alpha Model if args.model in ['Alpha', 'all']: - run_model('Alpha', Alpha(), run_date_day, args.log_level) + run_model('Alpha', Alpha(), aip_start_date, args.log_level) # Alpha 7 Model if args.model in ['Alpha7', 'all']: - run_model('Alpha7', Alpha7(), run_date_day, args.log_level) + run_model('Alpha7', Alpha7(), aip_start_date, args.log_level) # Prioritize New Model if args.model in ['Prioritize_New', 'all']: - run_model('Prioritize_New', New(), run_date_day, args.log_level) + run_model('Prioritize_New', New(), aip_start_date, args.log_level) # Prioritize Consistent Model if args.model in ['Prioritize_Consistent', 'all']: - run_model('Prioritize_Consistent', Consistent(), run_date_day, args.log_level) + run_model('Prioritize_Consistent', Consistent(), aip_start_date, args.log_level) # Prioritize Random Forest Model if args.model in ['Random_Forest', 'all']: - run_model('Random_Forest', RandomForest(), run_date_day, args.log_level) + run_model('Random_Forest', RandomForest(), aip_start_date, args.log_level) + except ValueError as err: logger.error(err) sys.exit() From cc38340f6ea774f239e20a7c7805b03f51b626da Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Tue, 12 Nov 2024 16:40:54 +0100 Subject: [PATCH 25/34] Add option to run all models between two dates (#76) * Rename date_day to start_date * Explicit calling parameters on run_model * Add end-date with default values * Validate start and end * Iterate through dates --- bin/aip | 85 ++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 28 deletions(-) diff --git a/bin/aip b/bin/aip index c431564..24454d9 100755 --- a/bin/aip +++ b/bin/aip @@ -33,6 +33,7 @@ import argparse import logging import sys from datetime import date +from datetime import timedelta from os import makedirs from os import path from aip.data.access import data_path @@ -45,7 +46,7 @@ from aip.utils.date_utils import validate_and_convert_date from aip.utils.knowledge_base import _rebuild -def run_model(aip_model_name, aip_model, date_day, log_level=logging.ERROR): +def run_model(aip_model_name, aip_model, start_date, log_level=logging.ERROR): """ Run a given model with exception handling """ @@ -56,8 +57,8 @@ def run_model(aip_model_name, aip_model, date_day, log_level=logging.ERROR): makedirs(model_output_dir) try: - blocklist = aip_model.run(date_day) - blocklist.to_csv(path.join(model_output_dir, f'AIP-{aip_model_name}-{str(date_day)}.csv.gz'), index=False, compression='gzip') + blocklist = aip_model.run(start_date) + blocklist.to_csv(path.join(model_output_dir, f'AIP-{aip_model_name}-{str(start_date)}.csv.gz'), index=False, compression='gzip') logging.info(f"{aip_model_name} model completed successfully.") except Exception as err: logging.error(f"Error running {aip_model_name} model: {err}") @@ -73,6 +74,7 @@ def main(): # Run Models command run_models_parser = subparsers.add_parser("run-models", help="Run models with optional date range.") run_models_parser.add_argument('--start-date', type=str, help='The start date for running the models (YYYY-MM-DD). Default is today.', default=str(date.today())) + run_models_parser.add_argument('--end-date', type=str, help='The end date for running the models (YYYY-MM-DD). Default is today.', default=str(date.today())) run_models_parser.add_argument('--model', type=str, choices=['Alpha', 'Alpha7', 'Prioritize_New', 'Prioritize_Consistent', 'Random_Forest', 'all'], default='all', help='Select AIP model to run. Default is all models.') # Rebuild KB command @@ -86,13 +88,17 @@ def main(): log_fmt = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' logging.basicConfig(level=args.log_level, format=log_fmt) + # If not set, default to today() + if not hasattr(args, 'start_date'): + args.start_date = date.today().strftime('%Y-%m-%d') + if not hasattr(args, 'end_date'): + args.end_date = date.today().strftime('%Y-%m-%d') + try: - # Validate start_date once not on every command - if hasattr(args,'start_date'): - aip_start_date = validate_and_convert_date(args.start_date) - else: - # If not set, default to today() - aip_start_date = date.today() + # Validate once not on every command + aip_start_date = validate_and_convert_date(args.start_date) + aip_end_date = validate_and_convert_date(args.end_date) + logger.debug(f"AIP will run models from start date {aip_start_date} to end date {aip_end_date}") # If model is not set, default to today if not hasattr(args,'model'): @@ -104,26 +110,49 @@ def main(): _rebuild(start_date=aip_start_date, log_level=args.log_level) sys.exit() - logging.info(f"Running {args.model} model(s) for date {aip_start_date}.") - # Run Alpha Model - if args.model in ['Alpha', 'all']: - run_model('Alpha', Alpha(), aip_start_date, args.log_level) - - # Alpha 7 Model - if args.model in ['Alpha7', 'all']: - run_model('Alpha7', Alpha7(), aip_start_date, args.log_level) - - # Prioritize New Model - if args.model in ['Prioritize_New', 'all']: - run_model('Prioritize_New', New(), aip_start_date, args.log_level) - - # Prioritize Consistent Model - if args.model in ['Prioritize_Consistent', 'all']: - run_model('Prioritize_Consistent', Consistent(), aip_start_date, args.log_level) - # Prioritize Random Forest Model - if args.model in ['Random_Forest', 'all']: - run_model('Random_Forest', RandomForest(), aip_start_date, args.log_level) + while aip_start_date <= aip_end_date: + logging.info(f"Running {args.model} model(s) for date {aip_start_date}.") + # Run Alpha Model + if args.model in ['Alpha', 'all']: + run_model(aip_model_name='Alpha', + aip_model=Alpha(), + start_date=aip_start_date, + log_level=args.log_level + ) + + # Alpha 7 Model + if args.model in ['Alpha7', 'all']: + run_model(aip_model_name='Alpha7', + aip_model=Alpha7(), + start_date=aip_start_date, + log_level=args.log_level + ) + + # Prioritize New Model + if args.model in ['Prioritize_New', 'all']: + run_model(aip_model_name='Prioritize_New', + aip_model=New(), + start_date=aip_start_date, + log_level=args.log_level + ) + + # Prioritize Consistent Model + if args.model in ['Prioritize_Consistent', 'all']: + run_model(aip_model_name='Prioritize_Consistent', + aip_model=Consistent(), + start_date=aip_start_date, + log_level=args.log_level + ) + + # Prioritize Random Forest Model + if args.model in ['Random_Forest', 'all']: + run_model(aip_model_name='Random_Forest', + aip_model=RandomForest(), + start_date=aip_start_date, + log_level=args.log_level + ) + aip_start_date += timedelta(days=1) except ValueError as err: logger.error(err) From 34db498ac6894bd2b0661ba5e8ffcb78bddea683 Mon Sep 17 00:00:00 2001 From: Veronica Valeros Date: Tue, 12 Nov 2024 17:19:27 +0100 Subject: [PATCH 26/34] Alpha(s) to raise exception if no data is available (#78) Validate data and raise exception --- lib/aip/models/alpha.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/aip/models/alpha.py b/lib/aip/models/alpha.py index db19e79..44ce497 100644 --- a/lib/aip/models/alpha.py +++ b/lib/aip/models/alpha.py @@ -46,11 +46,14 @@ def run(self, for_date=date.today()): # Get all the attackers IPs attacks = get_attacks(start, end, usecols=['orig']) attacks = pd.concat(attacks).drop_duplicates() - attacks = attacks.rename(columns={'orig':'ip'}) + if not attacks.empty: + attacks = attacks.rename(columns={'orig':'ip'}) - self.blocklist = attacks + self.blocklist = attacks - # Remove IPs from do_not_block_these_ips.csv - self.sanitize() + # Remove IPs from do_not_block_these_ips.csv + self.sanitize() - return self.blocklist + return self.blocklist + else: + raise ValueError("Please check data availability and try again.") From 18fe2180b4c92102746fbf49a4fb4ac47063aba6 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:53:48 +0200 Subject: [PATCH 27/34] Update README files to enhance setup instructions and clarify Docker usage --- README.md | 97 ++++++++++++++++++++++++++++---------------- etc/docker/README.md | 47 +++++++++++++-------- 2 files changed, 91 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index d00bf2d..65889b8 100644 --- a/README.md +++ b/README.md @@ -38,50 +38,75 @@ Each AIP model generates its own blocklist based on a specific criteria. The mai - A more experimental approach to increase blocklist efficiency. -## Installation +## Setup -AIP can be run through Docker (recommended) or natively. In any case, make sure the data is correctly accessible in the `data/raw/` folder and that the files `data/external/` are correctly configured. +AIP can run with Docker (recommended) or directly with Python. Either way, it needs two things: -### AIP Docker +1. **Zeek logs** in `data/raw/`, one folder per day (for example `data/raw/2024-10-22/conn.*`). With Docker you can point AIP to your Zeek logs folder instead of copying them. +2. **Your honeypot IPs** in `data/external/`. Copy the example files and edit them: -The best way to run AIP right now is using [Docker](etc/docker/README.md). + ```bash + cp data/external/honeypots_public_ips_example.csv data/external/honeypots_public_ips.csv + cp data/external/do_not_block_these_ips_example.csv data/external/do_not_block_these_ips.csv + ``` -AIP will automatically attempt to run all the models using the available data. Assuming the Zeek data is located in its usual location: + - `honeypots_public_ips.csv` (required): the public IPs of your honeypots. AIP only looks at traffic sent to these IPs. + - `do_not_block_these_ips.csv` (optional): IPs that must never appear in a blocklist. + +### Run with Docker + +From the AIP folder, assuming your Zeek logs are in `/opt/zeek/logs/`: + +```bash +docker run --rm \ + -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro \ + -v ${PWD}/data/:/home/aip/AIP/data/:rw \ + --name aip stratosphereips/aip:latest bin/aip +``` + +This runs all models for today. To use any of the [commands](#commands) below, write them after `bin/aip`. For example, to run all models for one day: ```bash -:~$ cd AIP -:~$ docker run --rm -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro -v ${PWD}/data/:/home/aip/AIP/data/:rw --name aip stratosphereips/aip:latest bin/aip +docker run --rm \ + -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro \ + -v ${PWD}/data/:/home/aip/AIP/data/:rw \ + --name aip stratosphereips/aip:latest bin/aip run-models --start-date 2026-10-12 --end-date 2026-10-13 ``` -### Local Installation - -To run AIP locally, follow the next steps: - -* Clone the AIP repository: - * `git clone https://github.com/stratosphereips/AIP.git ~/AIP` -* Access the AIP folder: - * `cd ~/AIP` -* Create a Python Virtual Environment to install the requirements - * `python3 -m venv aip-venv` -* Activate the virtual environment - * `source aip-venv/bin/activate` -* Install the Python requirements: - * `pip install -r requirements.txt` -* Set the PYTHONPATH environment variable so Python knows where to find all AIP components: - * `export PYTHONPATH=$(pwd)/lib` -* Let's test that AIP works with a simple test: - * `python3 bin/aip --help` - -### AIP Commands - -Get all the full list of options with `bin/aip --help`: - -- Run AIP for a specific day: - - `bin/aip -v run-models --start-date YYYY-MM-DD` -- Run AIP for a specific day and model: - - `bin/aip -v run-models --start-date YYYY-MM-DD --model Alpha` -- Rebuild AIP knowledge base from a specific date: - - `bin/aip -d rebuild-kb --start-date YYYY-MM-DD` +To build the image yourself, see the [Docker guide](etc/docker/README.md). + +### Run with Python + +```bash +git clone https://github.com/stratosphereips/AIP.git ~/AIP +cd ~/AIP +python3 -m venv aip-venv +source aip-venv/bin/activate +pip install -r requirements.txt +export PYTHONPATH=$(pwd)/lib +bin/aip --help +``` + +Run `source aip-venv/bin/activate` and the `export PYTHONPATH` line again each time you open a new terminal. + +## Commands + +| What you want | Command | +|---|---| +| Run all models for today | `bin/aip` | +| Run all models for one day | `bin/aip run-models --start-date 2026-10-02 --end-date 2026-10-02` | +| Run all models for several days | `bin/aip run-models --start-date 2026-10-01 --end-date 2026-10-12` | +| Run only one model | Add `--model` with `Alpha`, `Alpha7`, `Prioritize_New`, `Prioritize_Consistent` or `Random_Forest` | +| Rebuild the attacker history from a date until today | `bin/aip rebuild-kb --start-date 2026-10-01` | + +Things to know: + +- If you leave out `--end-date`, it is today. So `--start-date` alone runs every day from that date until today. +- The blocklist for a day is built from the attacks seen on the days before it. +- Blocklists are saved in `data/output//`. +- The attacker history is what the Prioritize and Random Forest models learn from. AIP builds it on its own as it runs. +- To see what AIP is doing, add `-v` (progress) or `-d` (full detail) right after `bin/aip`, for example `bin/aip -v run-models`. +- See all options with `bin/aip --help`. ## License diff --git a/etc/docker/README.md b/etc/docker/README.md index 88818af..5e4fccf 100644 --- a/etc/docker/README.md +++ b/etc/docker/README.md @@ -1,40 +1,51 @@ # Docker image for AIP -AIP docker aims to help in development and deployment of AIP algorithms to newcommers. The code of the repository is mounted and available from inside the docker image. The source of the data and the output folder of AIP is also mounted as a data volume inside the data/ folder to decouple from where this data really is in the host machine, easying the AIP deployment. +The Docker image includes AIP and everything it needs to run. Your Zeek logs and the AIP `data/` folder stay on your machine and are shared with the container when you run it. ## Build the image -To build the image, you can run the following command. - ```bash -:~$ git clone https://github.com/stratosphereips/AIP.git -:~$ cd AIP/ -:~/AIP$ docker build --build-arg uid=1000 --file etc/docker/Dockerfile --tag aip:latest . +git clone https://github.com/stratosphereips/AIP.git +cd AIP/ +docker build --build-arg uid=1000 --file etc/docker/Dockerfile --tag aip:latest . ``` ## Prepare the data -AIP needs raw network flow data to run. In this case, we assume you have Zeek logs in `/opt/zeek/logs`. +AIP reads Zeek logs. These steps assume your Zeek logs are in `/opt/zeek/logs/`. -Additionally, the following two files need to be edited and populated: -- `data/external/do_not_block_these_ips_example.csv`: you want to add here IPs that should not appear on the AIP blocklists -- `data/external/honeypots_public_ips_example.csv`: you want to add here the public IP of the honeypot or machine running Zeek +Copy the two example files and edit the copies: -First copy the files and then edit them: ```bash -:~/AIP$ cp data/external/do_not_block_these_ips_example.csv data/external/do_not_block_these_ips.csv -:~/AIP$ cp data/external/honeypots_public_ips_example.csv data/external/honeypots_public_ips.csv +cp data/external/honeypots_public_ips_example.csv data/external/honeypots_public_ips.csv +cp data/external/do_not_block_these_ips_example.csv data/external/do_not_block_these_ips.csv ``` +- `honeypots_public_ips.csv` (required): the public IPs of your honeypots. AIP only looks at traffic sent to these IPs. +- `do_not_block_these_ips.csv` (optional): IPs that must never appear in a blocklist. + ## Run the container -To run the container of that image you can run the following command: +Run all models for today: ```bash -:~/AIP$ docker run --rm -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro -v ${PWD}/data/:/home/aip/AIP/data/:rw --name aip aip:latest bin/aip +docker run --rm \ + -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro \ + -v ${PWD}/data/:/home/aip/AIP/data/:rw \ + --name aip aip:latest bin/aip ``` -An example output is shown below: +Run all models for one day, showing full detail with `-d`: + +```bash +docker run --rm \ + -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro \ + -v ${PWD}/data/:/home/aip/AIP/data/:rw \ + --name aip aip:latest bin/aip -d run-models --start-date 2024-10-22 --end-date 2024-10-22 +``` + +The output starts like this: + ``` 2024-10-23 13:02:36,513 - aip.data.access - DEBUG - Creating attacks for dates ['2024-10-22'] 2024-10-23 13:02:36,513 - aip.data.access - DEBUG - Making dataset from raw data for dates ['2024-10-22'] @@ -42,4 +53,6 @@ An example output is shown below: 2024-10-23 13:02:37,539 - aip.data.access - DEBUG - Writting file: /home/aip/AIP/data/processed/attacks.2024-10-22.csv.gz 2024-10-23 13:02:37,648 - aip.data.access - DEBUG - Creating attacks for dates ['2024-10-20', '2024-10-21'] ... -``` \ No newline at end of file +``` + +Without `-v` or `-d`, AIP only prints errors. Blocklists are saved in `data/output/`. See the [main README](../../README.md#commands) for all commands. From 6330b6c2bba67c3d2e676e394f6c2d2d76a437c9 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:57:25 +0200 Subject: [PATCH 28/34] Fix end date in Docker run command for model execution in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 65889b8..8fddfe9 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ This runs all models for today. To use any of the [commands](#commands) below, w docker run --rm \ -v /opt/zeek/logs/:/home/aip/AIP/data/raw:ro \ -v ${PWD}/data/:/home/aip/AIP/data/:rw \ - --name aip stratosphereips/aip:latest bin/aip run-models --start-date 2026-10-12 --end-date 2026-10-13 + --name aip stratosphereips/aip:latest bin/aip run-models --start-date 2026-10-12 --end-date 2026-10-12 ``` To build the image yourself, see the [Docker guide](etc/docker/README.md). From 10dc0fdd0f80eec19cbb8c0ca6874018579970f8 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:11:02 +0200 Subject: [PATCH 29/34] Update README to correct end date description for rebuilding attacker history --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8fddfe9..7d54fb2 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ Run `source aip-venv/bin/activate` and the `export PYTHONPATH` line again each t | Run all models for one day | `bin/aip run-models --start-date 2026-10-02 --end-date 2026-10-02` | | Run all models for several days | `bin/aip run-models --start-date 2026-10-01 --end-date 2026-10-12` | | Run only one model | Add `--model` with `Alpha`, `Alpha7`, `Prioritize_New`, `Prioritize_Consistent` or `Random_Forest` | -| Rebuild the attacker history from a date until today | `bin/aip rebuild-kb --start-date 2026-10-01` | +| Rebuild the attacker history from a date until yesterday | `bin/aip rebuild-kb --start-date 2026-10-01` | Things to know: From 3bcf52dd53cda1d48401949b3fcb823984fbddd9 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:11:29 +0200 Subject: [PATCH 30/34] Add date validation for knowledge base rebuild and model execution --- bin/aip | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/bin/aip b/bin/aip index 24454d9..f306df5 100755 --- a/bin/aip +++ b/bin/aip @@ -106,10 +106,17 @@ def main(): # Rebuild knowledge base if args.command == "rebuild-kb": + last_complete_day = date.today() - timedelta(days=1) + if aip_start_date > last_complete_day: + logger.error(f"The start date must be {last_complete_day} or earlier, today is not complete yet.") + sys.exit(1) logger.info(f"Rebuilding knowledge base from {aip_start_date} onwards") _rebuild(start_date=aip_start_date, log_level=args.log_level) sys.exit() + if aip_start_date > aip_end_date: + logger.error(f"The start date {aip_start_date} is after the end date {aip_end_date}.") + sys.exit(1) while aip_start_date <= aip_end_date: logging.info(f"Running {args.model} model(s) for date {aip_start_date}.") @@ -156,7 +163,7 @@ def main(): except ValueError as err: logger.error(err) - sys.exit() + sys.exit(1) if __name__ == '__main__': main() From 088613a70a6c9c8ca7c6e6a9e4671b6ef0445008 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:11:51 +0200 Subject: [PATCH 31/34] Refactor date validation logic for clarity and maintainability --- lib/aip/utils/date_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/aip/utils/date_utils.py b/lib/aip/utils/date_utils.py index a5f7936..ea6d408 100644 --- a/lib/aip/utils/date_utils.py +++ b/lib/aip/utils/date_utils.py @@ -8,9 +8,8 @@ def validate_and_convert_date(date_str): """ try: dateobj = datetime.strptime(date_str, '%Y-%m-%d').date() - date_today = date.today() - if dateobj > date_today: - raise ValueError(f"The input date '{date_str}' cannot be in the future.") - return dateobj except ValueError as err: - raise ValueError(f"Invalid date format: {date_str}, expected YYYY-MM-DD") from err \ No newline at end of file + raise ValueError(f"Invalid date format: {date_str}, expected YYYY-MM-DD") from err + if dateobj > date.today(): + raise ValueError(f"The input date '{date_str}' cannot be in the future.") + return dateobj From 9fa2e3e27396e9d30322669f8d3af0f86b6bff2b Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:12:03 +0200 Subject: [PATCH 32/34] Fix import paths for model modules in generate_historical_blocklists.py --- lib/aip/utils/generate_historical_blocklists.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/aip/utils/generate_historical_blocklists.py b/lib/aip/utils/generate_historical_blocklists.py index 6992bee..f55debf 100755 --- a/lib/aip/utils/generate_historical_blocklists.py +++ b/lib/aip/utils/generate_historical_blocklists.py @@ -30,9 +30,9 @@ from aip.data.access import data_path, project_dir from aip.models.alpha import Alpha from aip.models.alpha7 import Alpha7 -from lib.aip.models.prioritize_new import New -from lib.aip.models.prioritize_consistent import Consistent -from lib.aip.models.random_forest import RandomForest +from aip.models.prioritize_new import New +from aip.models.prioritize_consistent import Consistent +from aip.models.random_forest import RandomForest from aip.models.pareto import Pareto from datetime import date, timedelta from joblib import Parallel, delayed From 3b4a23ab023964817de280c6cd13a170636d8644 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:12:17 +0200 Subject: [PATCH 33/34] Update _rebuild function to set end_date to yesterday for complete data --- lib/aip/utils/knowledge_base.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/aip/utils/knowledge_base.py b/lib/aip/utils/knowledge_base.py index 8e970ab..754b150 100644 --- a/lib/aip/utils/knowledge_base.py +++ b/lib/aip/utils/knowledge_base.py @@ -75,14 +75,19 @@ def _build_knowledge(start, end, log_level=logging.ERROR): return last_knowledge -def _rebuild(start_date, end_date=date.today(), log_level=logging.ERROR): +def _rebuild(start_date, end_date=None, log_level=logging.ERROR): """ - Wrapper to rebuild the knowledge base from the specified start date to today. + Wrapper to rebuild the knowledge base from the specified start date to + yesterday, the last complete day. """ # Configure logger logger = logging.getLogger(__name__) logging.basicConfig(level=log_level) + # Today's data is incomplete, do not add it to the knowledge base + if end_date is None: + end_date = date.today() - timedelta(days=1) + start_time = time.time() logger.debug(f"_rebuild - Rebuilding knowledge base from {start_date} to {end_date}") From 11cec61608ad9c63a6f7432318d2d512dab25d22 Mon Sep 17 00:00:00 2001 From: Veronica Valeros <2458879+verovaleros@users.noreply.github.com> Date: Tue, 13 Oct 2026 10:12:27 +0200 Subject: [PATCH 34/34] Enhance future date validation test to check specific error message --- tests/test_lib_aip_utils_date_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_lib_aip_utils_date_utils.py b/tests/test_lib_aip_utils_date_utils.py index 35e8319..677e947 100644 --- a/tests/test_lib_aip_utils_date_utils.py +++ b/tests/test_lib_aip_utils_date_utils.py @@ -53,7 +53,7 @@ def test_future_date(self): # Test future date raises exception future_date = (date.today() + timedelta(days=1)).isoformat() - with self.assertRaises(ValueError): + with self.assertRaisesRegex(ValueError, "cannot be in the future"): validate_and_convert_date(future_date)