From ba5f07c65c07cf747f3e62b5f0d43df3b773a0d1 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Mon, 25 Aug 2025 13:35:24 +0200 Subject: [PATCH 01/79] Add commit author of 'commit_added' events to event info This allows for reconstruction of correct commit author if user is github Signed-off-by: Leo Sendelbach --- author_postprocessing/author_postprocessing.py | 8 ++++++-- issue_processing/issue_processing.py | 8 ++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index 2b54ef7..cea6efd 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -181,7 +181,7 @@ def is_github_noreply_author(name, email): commit_data_file = path.join(data_path, commits_list) commit_data = csv_writer.read_from_csv(commit_data_file) commit_hash_to_author = {commit[7]: commit[2:4] for commit in commit_data} - + author_name_to_data = {author[1]: author[1:3] for author in author_data_new} issue_data_new = [] for event in issue_data: @@ -189,12 +189,16 @@ def is_github_noreply_author(name, email): if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 commit_hash = event[12] - + name = event[13][1:-1] # extract commit author from commit data, if available if commit_hash in commit_hash_to_author: event[9] = commit_hash_to_author[commit_hash][0] event[10] = commit_hash_to_author[commit_hash][1] issue_data_new.append(event) + elif name in author_name_to_data: + event[9] = author_name_to_data[name][0] + event[10] = author_name_to_data[name][1] + issue_data_new.append(event) else: # the added commit is not part of the commit data. In most cases, this is due to merge commits # appearing in another pull request, as Codeface does not keep track of merge commits. As we diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 3db14d5..73d48e3 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -364,6 +364,7 @@ def merge_issue_events(issue_data): # it is a commit which was added to the pull request if rel_commit["type"] == "commitAddedToPullRequest": rel_commit["event"] = "commit_added" + rel_commit["event_info_2"] = rel_commit["commit"]["author"] # if the related commit was mentioned in an issue comment: elif rel_commit["type"] == "commitMentionedInIssue": @@ -750,6 +751,9 @@ def get_user_from_id(idx, buffer_db=user_buffer): for event in issue["eventsList"]: event["user"] = get_id_and_update_user(event["user"]) + if event["event"] == "commit_added": + event["event_info_2"] = get_id_and_update_user(event["event_info_2"]) + # check database for the reference-target user if needed if event["ref_target"] != "": event["ref_target"] = get_id_and_update_user(event["ref_target"]) @@ -763,6 +767,10 @@ def get_user_from_id(idx, buffer_db=user_buffer): for event in issue["eventsList"]: event["user"] = get_user_from_id(event["user"]) + # for commit_added events, save the commit's author's name in event_info_2 + if event["event"] == "commit_added": + event["event_info_2"] = get_user_from_id(event["event_info_2"])["name"] + # get the reference-target user if needed if event["ref_target"] != "": event["ref_target"] = get_user_from_id(event["ref_target"]) From c40df30e919b3877aea11058c2fa917e34443057 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 26 Aug 2025 10:56:56 +0200 Subject: [PATCH 02/79] Update Copyright headers also added one comment for clarity Signed-off-by: Leo Sendelbach --- author_postprocessing/author_postprocessing.py | 2 ++ issue_processing/issue_processing.py | 1 + 2 files changed, 3 insertions(+) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index cea6efd..a7a5488 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -14,6 +14,7 @@ # # Copyright 2015-2017 by Claus Hunsen # Copyright 2020-2022 by Thomas Bock +# Copyright 2025 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -189,6 +190,7 @@ def is_github_noreply_author(name, email): if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 commit_hash = event[12] + # extract author name from event info 2 while cutting excess '"' name = event[13][1:-1] # extract commit author from commit data, if available if commit_hash in commit_hash_to_author: diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 73d48e3..53c8313 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -18,6 +18,7 @@ # Copyright 2018-2019 by Anselm Fehnker # Copyright 2019 by Thomas Bock # Copyright 2020-2021 by Thomas Bock +# Copyright 2025 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. From eb1849b75cea1f4e17746f4f18100f1f393c4eef Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Thu, 25 Sep 2025 14:37:37 +0200 Subject: [PATCH 03/79] Add connected events reconstruction also save merge commits reconstruction of connected events is done by first saving all connected events that occured at the same time. Then, it is possible to match connected events iff: - half of the involved issues are equal, meaning that one issue is connected to multiple others - half rounded up of the involved isses are equal, meaning that we have one external connected event and then the previous case with the remaining issues Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 97 ++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 53c8313..a09b765 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -31,6 +31,7 @@ import os import sys from datetime import datetime, timedelta +import math from logging import getLogger from codeface_utils.cluster.idManager import dbIdManager, csvIdManager @@ -56,6 +57,9 @@ # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" +filtered_connected_events = dict() +external_connected_events = dict() + def run(): # get all needed paths and arguments for the method call. parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') @@ -297,6 +301,7 @@ def merge_issue_events(issue_data): log.info("Merge issue events ...") issue_data_to_update = dict() + connected_events = dict() for issue in issue_data: @@ -493,6 +498,28 @@ def merge_issue_events(issue_data): event["ref_target"] = event["user"] event["user"] = event["assigner"] + # if event is merged event, save the hash of the merge commit in event_info_1 + if event["event"] == "merged": + event["event_info_1"] = event["commit"]["hash"] + + # if event is connected event, create or add to a matching dict entry by matching timestamps, for later reconstruction + if event["event"] == "connected": + if event["created_at"] in connected_events.keys() and connected_events[event["created_at"]]["user"] == event["user"]: + connected_events[event["created_at"]]["issues"].append(issue["number"]) + elif subtract_seconds_from_time(event["created_at"], 1) in connected_events.keys() \ + and connected_events[subtract_seconds_from_time(event["created_at"], 1)]["user"] == event["user"]: + connected_events[subtract_seconds_from_time(event["created_at"], 1)]["issues"].append(issue["number"]) + event["created_at"] = subtract_seconds_from_time(event["created_at"], 1) + elif subtract_seconds_from_time(event["created_at"], -1) in connected_events.keys() \ + and connected_events[subtract_seconds_from_time(event["created_at"], -1)]["user"] == event["user"]: + connected_events[subtract_seconds_from_time(event["created_at"], -1)]["issues"].append(issue["number"]) + event["created_at"] = subtract_seconds_from_time(event["created_at"], -1) + else: + connected_info = dict() + connected_info["issues"] = [issue["number"]] + connected_info["user"] = issue["user"] + connected_events[event["created_at"]] = connected_info + # merge events, relatedCommits, relatedIssues and comment lists issue["eventsList"] = issue["commentsList"] + issue["eventsList"] + issue["relatedIssues"] + issue[ "relatedCommits"] + issue["reviewsList"] @@ -504,6 +531,10 @@ def merge_issue_events(issue_data): # sorts eventsList by time issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) + # filter out connected events which cannot be perfectly matched + global filtered_connected_events + filtered_connected_events = dict(filter(lambda item: filter_connected_events(item[0], item[1]), connected_events.iteritems())) + # updates all the issues by the temporarily stored referenced_by events for _, value in issue_data_to_update.items(): for issue in issue_data: @@ -513,6 +544,41 @@ def merge_issue_events(issue_data): return issue_data +def filter_connected_events(key, value): + num_issues = len(value["issues"]) + global external_connected_events + # if only a single connected event exists at this time, it has to be connecting to an external issue + if num_issues == 1: + external_connected_events[key] = value + return False + # if 2 connected events exist, matching them is trivial + if num_issues == 2: + return True + occurances = {x: value["issues"].count(x) for x in set(value["issues"])} + # otherwise, if it is an even number, check if it can be easily matched, + # meaning that exactly half the events occur in the same issue + if num_issues % 2 == 0 and num_issues/2 in occurances.values(): + # duplicate issue list for matching the issues later + value["multi_issues_copy"] = list(value["issues"]) + return True + # if it is an odd number, check if it can be easily matched + # meaning that exactly half (rounded up) the events occur in the same issue + if num_issues % 2 == 1 and math.ceil(num_issues/2) in occurances.values(): + for sub_key, sub_value in occurances.iteritems(): + # then, assign one of them as an external connected event and proceed as in previous case + if sub_value == math.ceil(num_issues/2): + new_entry = dict() + new_entry["user"] = value["user"] + new_entry["issues"] = [sub_key] + external_connected_events[key] = new_entry + value["issues"].remove(sub_key) + # duplicate issue list for matching the issues later + value["multi_issues_copy"] = list(value["issues"]) + return True + # no other variants can be easily matched + return False + + def reformat_events(issue_data): """ Re-format event information dependent on the event type. @@ -543,6 +609,37 @@ def reformat_events(issue_data): if event["ref_target"] is not None and not event["ref_target"] == "": users = update_user_dict(users, event["ref_target"]) + # reconstruction of connections + if event["event"] == "connected": + external = False + # check if event is external + for key, value in external_connected_events.iteritems(): + if issue["number"] in value["issues"]: + if key == event["created_at"]: + external = True + event["event_info_1"] = "external" + value["issues"].remove(issue["number"]) + # if so, skip the next checks + if external: + continue + # otherwise, it must be internal + for key, value in filtered_connected_events.iteritems(): + if issue["number"] in value["issues"]: + if key == event["created_at"]: + if len(value["issues"]) == 2: + # if only 2 events occured at this timestamp, matching the issues is trivial + event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] + else: + occurances = {x: value["issues"].count(x) for x in set(value["issues"])} + if occurances[issue["number"]] == max(occurances.values()): + # otherwise, if current issue is the centerpiece of all connected events, use previous copy to match issues + number = next(x for x in value["multi_issues_copy"] if x != issue["number"]) + value["multi_issues_copy"].remove(number) + event["event_info_1"] = number + else: + # if current issue is not the centerpiece, connect it to the centerpiece + event["event_info_1"] = max(occurances, key = occurances.get) + # as the user dictionary is created, start re-formating the event information of all issues for issue in issue_data: From dd3f1516a35adc680195336fe5ca3190662b250d Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 14 Oct 2025 14:43:33 +0200 Subject: [PATCH 04/79] Remove unnecessary returns of issue data since data is modified in-place, return of input data is not needed Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index a09b765..64e4253 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -82,13 +82,13 @@ def run(): # 1) load the list of issues issues = load(__srcdir) # 2) re-format the issues - issues = reformat_issues(issues) + reformat_issues(issues) # 3) merges all issue events into one list - issues = merge_issue_events(issues) + merge_issue_events(issues) # 4) re-format the eventsList of the issues - issues = reformat_events(issues) + reformat_events(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) + insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) @@ -287,7 +287,7 @@ def reformat_issues(issue_data): else: issue["type"].append("issue") - return issue_data + return def merge_issue_events(issue_data): @@ -541,7 +541,7 @@ def merge_issue_events(issue_data): if issue["number"] == value["number"]: issue["eventsList"] = issue["eventsList"] + value["eventsList"] - return issue_data + return def filter_connected_events(key, value): @@ -750,7 +750,7 @@ def reformat_events(issue_data): for event_to_remove in events_to_remove: issue["eventsList"].remove(event_to_remove) - return issue_data + return def insert_user_data(issues, conf, resdir): @@ -889,7 +889,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): username_dump = os.path.join(resdir, "usernames.list") csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) - return issues + return def print_to_disk(issues, results_folder): From 62ebd6dd5d1fe850fedcd94ff398b5d3587f416d Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 14 Oct 2025 14:47:19 +0200 Subject: [PATCH 05/79] Add reasons to reopen/closed events ALso add commit hash if closed by commit Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 64e4253..0af1306 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -657,13 +657,16 @@ def reformat_events(issue_data): if event["event"] == "closed": event["event"] = "state_updated" event["event_info_1"] = "closed" # new state - event["event_info_2"] = "open" # old state + if event["commit"] is not None: + event["event_info_2"] = event["commit"]["hash"] + else: + event["event_info_2"] = event["state_reason"] issue["state_new"] = "closed" elif event["event"] == "reopened": event["event"] = "state_updated" event["event_info_1"] = "open" # new state - event["event_info_2"] = "closed" # old state + event["event_info_2"] = event["state_reason"] issue["state_new"] = "reopened" elif event["event"] == "labeled": From 51eee0e177bc456e3b98858425054b1223f8a33f Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 14 Oct 2025 17:01:48 +0200 Subject: [PATCH 06/79] Add GitHub issue types also rename 'new feature' to 'feature' Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 7 +++++-- issue_processing/jira_issue_processing.py | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 0af1306..720ffc2 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -46,7 +46,7 @@ log = getLogger(__name__) # known types from JIRA and GitHub default labels -known_types = {"bug", "improvement", "enhancement", "new feature", "task", "test", "wish"} +known_types = {"bug", "improvement", "enhancement", "feature", "task", "test", "wish"} # known resolutions from JIRA and GitHub default labels known_resolutions = {"unresolved", "fixed", "wontfix", "duplicate", "invalid", "incomplete", "cannot reproduce", @@ -246,7 +246,10 @@ def reformat_issues(issue_data): for issue in issue_data: # empty container for issue types - issue["type"] = [] + if issue["type"] is None: + issue["type"] = [] + else: + issue["type"] = [issue["type"]["name"].lower()] # empty container for issue resolutions issue["resolution"] = [] diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index 4220b96..ee3ae62 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -303,9 +303,12 @@ def parse_xml(issue_data, persons, skip_history, referenced_bys): link = issue_x.getElementsByTagName("link")[0] issue["url"] = link.firstChild.data - type = issue_x.getElementsByTagName("type")[0] - issue["type"] = type.firstChild.data - issue["type_list"] = ["issue", str(type.firstChild.data.lower())] + type = issue_x.getElementsByTagName("type")[0].firstChild.data + # rename 'new feature' type to 'feature' to be in line with the github original issue type + if type == "New Feature": + type = "Feature" + issue["type"] = type + issue["type_list"] = ["issue", str(type.lower())] status = issue_x.getElementsByTagName("status")[0] issue["state"] = status.firstChild.data From 1ef9df845f1ba58ed14707911cec85747498282d Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 14 Oct 2025 17:26:55 +0200 Subject: [PATCH 07/79] Simplify loops for reconstruction of connections also remove duplicates from type list Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 67 +++++++++++----------------- 1 file changed, 26 insertions(+), 41 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 720ffc2..d35ba47 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -57,9 +57,6 @@ # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" -filtered_connected_events = dict() -external_connected_events = dict() - def run(): # get all needed paths and arguments for the method call. parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') @@ -84,9 +81,10 @@ def run(): # 2) re-format the issues reformat_issues(issues) # 3) merges all issue events into one list - merge_issue_events(issues) + external_connected_events = dict() + filtered_connected_events = merge_issue_events(issues, external_connected_events) # 4) re-format the eventsList of the issues - reformat_events(issues) + reformat_events(issues, filtered_connected_events, external_connected_events) # 5) update user data with Codeface database and dump username-to-name/e-mail list insert_user_data(issues, __conf, __resdir) # 6) dump result to disk @@ -293,7 +291,7 @@ def reformat_issues(issue_data): return -def merge_issue_events(issue_data): +def merge_issue_events(issue_data, external_connected_events): """ All issue events are merged together in the eventsList. This simplifies processing in later steps. @@ -535,8 +533,7 @@ def merge_issue_events(issue_data): issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) # filter out connected events which cannot be perfectly matched - global filtered_connected_events - filtered_connected_events = dict(filter(lambda item: filter_connected_events(item[0], item[1]), connected_events.iteritems())) + filtered_connected_events = dict(filter(lambda item: filter_connected_events(item[0], item[1], external_connected_events), connected_events.items())) # updates all the issues by the temporarily stored referenced_by events for _, value in issue_data_to_update.items(): @@ -544,12 +541,11 @@ def merge_issue_events(issue_data): if issue["number"] == value["number"]: issue["eventsList"] = issue["eventsList"] + value["eventsList"] - return + return filtered_connected_events -def filter_connected_events(key, value): +def filter_connected_events(key, value, external_connected_events): num_issues = len(value["issues"]) - global external_connected_events # if only a single connected event exists at this time, it has to be connecting to an external issue if num_issues == 1: external_connected_events[key] = value @@ -582,7 +578,7 @@ def filter_connected_events(key, value): return False -def reformat_events(issue_data): +def reformat_events(issue_data, filtered_connected_events, external_connected_events): """ Re-format event information dependent on the event type. @@ -614,34 +610,23 @@ def reformat_events(issue_data): # reconstruction of connections if event["event"] == "connected": - external = False - # check if event is external - for key, value in external_connected_events.iteritems(): - if issue["number"] in value["issues"]: - if key == event["created_at"]: - external = True - event["event_info_1"] = "external" - value["issues"].remove(issue["number"]) - # if so, skip the next checks - if external: - continue - # otherwise, it must be internal - for key, value in filtered_connected_events.iteritems(): - if issue["number"] in value["issues"]: - if key == event["created_at"]: - if len(value["issues"]) == 2: - # if only 2 events occured at this timestamp, matching the issues is trivial - event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] - else: - occurances = {x: value["issues"].count(x) for x in set(value["issues"])} - if occurances[issue["number"]] == max(occurances.values()): - # otherwise, if current issue is the centerpiece of all connected events, use previous copy to match issues - number = next(x for x in value["multi_issues_copy"] if x != issue["number"]) - value["multi_issues_copy"].remove(number) - event["event_info_1"] = number - else: - # if current issue is not the centerpiece, connect it to the centerpiece - event["event_info_1"] = max(occurances, key = occurances.get) + if event["created_at"] in external_connected_events \ + and issue["number"] in external_connected_events[event["created_at"]]["issues"]: + event["event_info_1"] = "external" + external_connected_events[event["created_at"]]["issues"].remove(issue["number"]) + elif event["created_at"] in filtered_connected_events \ + and issue["number"] in filtered_connected_events[event["created_at"]]["issues"]: + value = filtered_connected_events[event["created_at"]] + if len(value["issues"]) == 2: + event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] + else: + occurances = {x: value["issues"].count(x) for x in set(value["issues"])} + if occurances[issue["number"]] == max(occurances.values()): + number = next(x for x in value["multi_issues_copy"] if x != issue["number"]) + value["multi_issues_copy"].remove(number) + event["event_info_1"] = number + else: + event["event_info_1"] = max(occurances, key = occurances.get) # as the user dictionary is created, start re-formating the event information of all issues for issue in issue_data: @@ -677,7 +662,7 @@ def reformat_events(issue_data): event["event_info_1"] = label # if the label is in this list, it also is a type of the issue - if label in known_types: + if label in known_types and label not in issue["type"]: issue["type"].append(str(label)) # creates an event for type updates and adds it to the eventsList From 5632b9d837f7df3d390ee23d905b768844a5d60e Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 14 Oct 2025 17:28:23 +0200 Subject: [PATCH 08/79] Add subissues to results csv using empty line reserved for jira components Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index d35ba47..3cfe110 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -907,7 +907,7 @@ def print_to_disk(issues, results_folder): json.dumps(issue["resolution"]), issue["created_at"], issue["closed_at"], - json.dumps([]), # components + json.dumps([issue["subIssues"]]), # components event["event"], event["user"]["name"], event["user"]["email"], From 4690c68e9f79cfab30ef59c972e7c282caeb5113 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 21 Oct 2025 14:10:18 +0200 Subject: [PATCH 09/79] Remove unneccesary return value also added copyright header Signed-off-by: Leo Sendelbach --- issue_processing/jira_issue_processing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index ee3ae62..a431ffa 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -19,6 +19,7 @@ # Copyright 2020-2021 by Thomas Bock # Copyright 2026 by Thomas Bock # Copyright 2023, 2025 by Maximilian Löffler +# Copyright 2025 by Leo Sendelbach # All Rights Reserved. """ This file is able to extract Jira issue data from xml files. @@ -128,7 +129,7 @@ def run(): referenced_issue["history"].append(referenced_by) # 5) update user data with Codeface database - processed_issues = insert_user_data(processed_issues, __conf) + insert_user_data(processed_issues, __conf) # 6) dump result to disk print_to_disk(processed_issues, __resdir) # # 7) export for Gephi @@ -695,7 +696,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): event["event_info_2"] = assigned_user["email"] log.debug("number of issues after insert_user_data: '{}'".format(len(issues))) - return issues + return def print_to_disk(issues, results_folder): From f1e93d3f40416184ff2cfa61c88bc787076406b2 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 21 Oct 2025 14:11:29 +0200 Subject: [PATCH 10/79] Add comments also minor fixes and removal of math.ceil Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 36 +++++++++++++++++++++------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 3cfe110..51f3ce0 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -80,10 +80,15 @@ def run(): issues = load(__srcdir) # 2) re-format the issues reformat_issues(issues) - # 3) merges all issue events into one list + # create an empty dict for external connected events, meaning connected + # events that connect to an issue in another repository external_connected_events = dict() + # 3) merges all issue events into one list + # this step returns a dict containing all connected events that can be matched to the correct issues later filtered_connected_events = merge_issue_events(issues, external_connected_events) # 4) re-format the eventsList of the issues + # this step also reconstructs the connections previously stored + # in 'external_connected_events' and 'filtered_connected_events' reformat_events(issues, filtered_connected_events, external_connected_events) # 5) update user data with Codeface database and dump username-to-name/e-mail list insert_user_data(issues, __conf, __resdir) @@ -506,16 +511,20 @@ def merge_issue_events(issue_data, external_connected_events): # if event is connected event, create or add to a matching dict entry by matching timestamps, for later reconstruction if event["event"] == "connected": if event["created_at"] in connected_events.keys() and connected_events[event["created_at"]]["user"] == event["user"]: + # if there is already a connected event at this time by this user, add this event to the list connected_events[event["created_at"]]["issues"].append(issue["number"]) elif subtract_seconds_from_time(event["created_at"], 1) in connected_events.keys() \ and connected_events[subtract_seconds_from_time(event["created_at"], 1)]["user"] == event["user"]: + # same as above, but accounting for a possible difference in timestamps of 1 second between matching events connected_events[subtract_seconds_from_time(event["created_at"], 1)]["issues"].append(issue["number"]) event["created_at"] = subtract_seconds_from_time(event["created_at"], 1) elif subtract_seconds_from_time(event["created_at"], -1) in connected_events.keys() \ and connected_events[subtract_seconds_from_time(event["created_at"], -1)]["user"] == event["user"]: + # same as above, with offset calculated in the other direction connected_events[subtract_seconds_from_time(event["created_at"], -1)]["issues"].append(issue["number"]) event["created_at"] = subtract_seconds_from_time(event["created_at"], -1) else: + # if there is no connected event yet at this timestamp, create a new entry for this event connected_info = dict() connected_info["issues"] = [issue["number"]] connected_info["user"] = issue["user"] @@ -553,19 +562,19 @@ def filter_connected_events(key, value, external_connected_events): # if 2 connected events exist, matching them is trivial if num_issues == 2: return True - occurances = {x: value["issues"].count(x) for x in set(value["issues"])} + occurences = {x: value["issues"].count(x) for x in set(value["issues"])} # otherwise, if it is an even number, check if it can be easily matched, # meaning that exactly half the events occur in the same issue - if num_issues % 2 == 0 and num_issues/2 in occurances.values(): + if num_issues % 2 == 0 and num_issues/2 in occurences.values(): # duplicate issue list for matching the issues later value["multi_issues_copy"] = list(value["issues"]) return True # if it is an odd number, check if it can be easily matched # meaning that exactly half (rounded up) the events occur in the same issue - if num_issues % 2 == 1 and math.ceil(num_issues/2) in occurances.values(): - for sub_key, sub_value in occurances.iteritems(): + if num_issues % 2 == 1 and (num_issues + 1)/2 in occurences.values(): + for sub_key, sub_value in occurences.iteritems(): # then, assign one of them as an external connected event and proceed as in previous case - if sub_value == math.ceil(num_issues/2): + if sub_value == (num_issues + 1)/2: new_entry = dict() new_entry["user"] = value["user"] new_entry["issues"] = [sub_key] @@ -612,21 +621,30 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev if event["event"] == "connected": if event["created_at"] in external_connected_events \ and issue["number"] in external_connected_events[event["created_at"]]["issues"]: + # if the event is an external connected event, mark it as such and remove this issue from the list event["event_info_1"] = "external" external_connected_events[event["created_at"]]["issues"].remove(issue["number"]) elif event["created_at"] in filtered_connected_events \ and issue["number"] in filtered_connected_events[event["created_at"]]["issues"]: + # if it is instead an internal connected event value = filtered_connected_events[event["created_at"]] if len(value["issues"]) == 2: + # and we only have 2 issues in the list, connect to the other issue event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] else: - occurances = {x: value["issues"].count(x) for x in set(value["issues"])} - if occurances[issue["number"]] == max(occurances.values()): + # and we have more than two issues, count each issue's occurences + occurences = {x: value["issues"].count(x) for x in set(value["issues"])} + if occurences[issue["number"]] == max(occurences.values()): + # if our issue is the most common one, that means it is the common denominator + # for all connected events at this time + # so this event connects to any other issue + # which is then removed from a copied list to avoid duplications number = next(x for x in value["multi_issues_copy"] if x != issue["number"]) value["multi_issues_copy"].remove(number) event["event_info_1"] = number else: - event["event_info_1"] = max(occurances, key = occurances.get) + # otherwise, connect this event to the common denominator + event["event_info_1"] = max(occurences, key=occurences.get) # as the user dictionary is created, start re-formating the event information of all issues for issue in issue_data: From 8351b311d55d3ab25acc8fb50a8ae19cae155b5f Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Fri, 31 Oct 2025 16:28:42 +0100 Subject: [PATCH 11/79] Add new json field for suggestions to result comments now each have a boolean field that describes whether the comment contains a suggestion or not Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 51f3ce0..4e5a901 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -745,7 +745,10 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev # "state_new" and "resolution" of the issue give the information about the state and the resolution of # the issue when the comment was written, because the eventsList is sorted by time event["event_info_1"] = issue["state_new"] - event["event_info_2"] = issue["resolution"] + if "contains_suggestion" in event: + event["event_info_2"] = event["contains_suggestion"] + else: + event["event_info_2"] = False elif event["event"] == "referenced" and event["commit"] is not None: # remove "referenced" events originating from commits From fa67649ce4557bd92b1bedae261dfa5249926428 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Fri, 31 Oct 2025 16:36:11 +0100 Subject: [PATCH 12/79] Improve documentation dicts for reconstructing connected events are now better explained and the comments do not disruot the workflow in the run function anymore Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 4e5a901..bc64c3b 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -80,15 +80,10 @@ def run(): issues = load(__srcdir) # 2) re-format the issues reformat_issues(issues) - # create an empty dict for external connected events, meaning connected - # events that connect to an issue in another repository - external_connected_events = dict() # 3) merges all issue events into one list - # this step returns a dict containing all connected events that can be matched to the correct issues later + external_connected_events = dict() filtered_connected_events = merge_issue_events(issues, external_connected_events) # 4) re-format the eventsList of the issues - # this step also reconstructs the connections previously stored - # in 'external_connected_events' and 'filtered_connected_events' reformat_events(issues, filtered_connected_events, external_connected_events) # 5) update user data with Codeface database and dump username-to-name/e-mail list insert_user_data(issues, __conf, __resdir) @@ -542,6 +537,8 @@ def merge_issue_events(issue_data, external_connected_events): issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) # filter out connected events which cannot be perfectly matched + # and populate external_connected_events dict + # because this happens in place, we do not need to return the external_connected_event dict later filtered_connected_events = dict(filter(lambda item: filter_connected_events(item[0], item[1], external_connected_events), connected_events.items())) # updates all the issues by the temporarily stored referenced_by events @@ -550,6 +547,7 @@ def merge_issue_events(issue_data, external_connected_events): if issue["number"] == value["number"]: issue["eventsList"] = issue["eventsList"] + value["eventsList"] + # return the filtered_connected_events dict for later reconstruction return filtered_connected_events From a9eed8abd536dd8feb917e3d062e7260d0c7c1df Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 4 Nov 2025 12:46:51 +0100 Subject: [PATCH 13/79] Incorporate requested changes includes: - updated comments - spelling mistake - fix for potential crash if script is used on old data Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index bc64c3b..1dbddea 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -264,7 +264,7 @@ def reformat_issues(issue_data): if issue["relatedCommits"] is None: issue["relatedCommits"] = [] - # if an issue has no reviewsList, an empty Listgets created + # if an issue has no reviewsList, an empty List gets created if issue["reviewsList"] is None: issue["reviewsList"] = [] @@ -272,6 +272,10 @@ def reformat_issues(issue_data): if "relatedIssues" not in issue: issue["relatedIssues"] = [] + # if an issue has no sub-issue list, an empty List gets created + if "subIssues" not in issue: + issue["subIssues"] = [] + # add "closed_at" information if not present yet if issue["closed_at"] is None: issue["closed_at"] = "" @@ -740,9 +744,10 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev issue["eventsList"].append(resolution_event) elif event["event"] == "commented": - # "state_new" and "resolution" of the issue give the information about the state and the resolution of + # "state_new" of the issue gives the information about the state of # the issue when the comment was written, because the eventsList is sorted by time event["event_info_1"] = issue["state_new"] + # if event is a review comment, it can contain suggestions if "contains_suggestion" in event: event["event_info_2"] = event["contains_suggestion"] else: From 8066db9932d9632e63ca5b062ce1ebf7996674b2 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 27 Jan 2026 13:41:27 +0100 Subject: [PATCH 14/79] Add copilot user unification to author postprocessing author postprocessing now also contains a list of known copilot use names that can be extended to unify more different copilot users Signed-off-by: Leo Sendelbach --- author_postprocessing/author_postprocessing.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index a7a5488..ac83b69 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -14,7 +14,7 @@ # # Copyright 2015-2017 by Claus Hunsen # Copyright 2020-2022 by Thomas Bock -# Copyright 2025 by Leo Sendelbach +# Copyright 2025-2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -54,6 +54,15 @@ setup_logging() log = getLogger(__name__) +## +# GLOBAL VARIABLES +## + +# global variable containing all known copilot users and the name and mail adress copilot users will be assigned +known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agentbot"} +copilot_unified_name = "Copilot" +copilot_unified_email = "copilot@example.com" + ## # RUN POSTPROCESSING ## @@ -82,7 +91,7 @@ def perform_data_backup(results_path, results_path_backup): copy(current_file, backup_file) -def fix_github_browser_commits(data_path, issues_github_list, commits_list, authors_list, emails_list, bots_list): +def fix_github_browser_commits(data_path, issues_github_list, commits_list, authors_list, emails_list, bots_list, unify_copilot_users=True): """ Replace the author "GitHub " in both commit and GitHub issue data by the correct author. The author "GitHub " is automatically inserted as the committer of a commit that is made when @@ -186,6 +195,11 @@ def is_github_noreply_author(name, email): issue_data_new = [] for event in issue_data: + # unify events to use a single copilot user for all events triggered by a known copilot user + if unify_copilot_users and event[9] in known_copilot_users: + event[9] = copilot_unified_name + event[10] = copilot_unified_email + # replace author if necessary if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 From eb78dbaf6d78a7ea137401abbf47bbad7637907a Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 27 Jan 2026 13:48:43 +0100 Subject: [PATCH 15/79] Assign copilot user data in case of specific events the events 'copilot_work_started' and 'copilot_work_finished' now always have the standard copilot user data Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 1dbddea..b33c88a 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -18,7 +18,7 @@ # Copyright 2018-2019 by Anselm Fehnker # Copyright 2019 by Thomas Bock # Copyright 2020-2021 by Thomas Bock -# Copyright 2025 by Leo Sendelbach +# Copyright 2025-2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -57,6 +57,9 @@ # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" +# Copilot username to be assigned in specific copilot events +copilot_username = "Copilot" + def run(): # get all needed paths and arguments for the method call. parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') @@ -491,6 +494,12 @@ def merge_issue_events(issue_data, external_connected_events): if event["event"] == "review_requested" or event["event"] == "review_request_removed": event["ref_target"] = event["requestedReviewer"] + # if event is a specific copilot event, assign the copilot user data + if event["event"] == "copilot_work_started" or event["event"] == "copilot_work_finished": + event["user"]["name"] = None + event["user"]["username"] = copilot_username + event["user"]["email"] = "" + # if event dismisses a review, we can determine the original state of the corresponding review if event["event"] == "review_dismissed": for review in issue["reviewsList"]: From a0ebc1437e04a5ee1bf1f2f6d8a7a967f4b6b0f5 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 27 Jan 2026 15:27:23 +0100 Subject: [PATCH 16/79] Add documentation for new copilot user unification Method doc updated to reflect new functionality Signed-off-by: Leo Sendelbach --- author_postprocessing/author_postprocessing.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index ac83b69..d7fcc6b 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -102,7 +102,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth "GitHub " are removed. Also "mentioned" or "subscribed" events in the GitHub issue data which reference the author "GitHub " are removed from the GitHub issue data. In addition, remove the author "GitHub " also from the author data and bot data and remove e-mails that have been sent - by this author. + by this author. This method also unifies all known copilot users into a single user if desired. :param data_path: the path to the project data that is to be fixed :param issues_github_list: file name of the github issue data @@ -110,6 +110,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth :param authors_list: file name of the corresponding author data :param emails_list: file name of the corresponding email data :param bots_list: file name of the corresponding bot data + :param unify_copilot_users: whether to unify known copilot users into a single user """ github_user = "GitHub" github_email = "noreply@github.com" From e496e66ae8619bdbe3d2c091d03f0fbc3d9271f1 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 27 Jan 2026 15:48:39 +0100 Subject: [PATCH 17/79] Fix connected event assignment previously, the creator of the issues was falsely matched to the connected event instead of the user triggering the event Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index b33c88a..bd1c191 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -535,7 +535,7 @@ def merge_issue_events(issue_data, external_connected_events): # if there is no connected event yet at this timestamp, create a new entry for this event connected_info = dict() connected_info["issues"] = [issue["number"]] - connected_info["user"] = issue["user"] + connected_info["user"] = event["user"] connected_events[event["created_at"]] = connected_info # merge events, relatedCommits, relatedIssues and comment lists From 53e3b0f29e3163bd42afdf5d2703d114e03b09a1 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Wed, 18 Feb 2026 14:06:28 +0100 Subject: [PATCH 18/79] Unify copilot users in all files unification now done on all files, which should prevent any issues arising from unknown authors during anonymization also move all global variables to a new utils file Signed-off-by: Leo Sendelbach --- .../author_postprocessing.py | 69 +++++++++++-------- github_user_utils/github_user_utils.py | 54 +++++++++++++++ issue_processing/issue_processing.py | 5 +- 3 files changed, 95 insertions(+), 33 deletions(-) create mode 100644 github_user_utils/github_user_utils.py diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index d7fcc6b..e908a2c 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -53,15 +53,10 @@ # create logger setup_logging() log = getLogger(__name__) +from github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ + is_github_noreply_author, github_user, github_email, \ + commit_added_event, mentioned_event, subscribed_event -## -# GLOBAL VARIABLES -## - -# global variable containing all known copilot users and the name and mail adress copilot users will be assigned -known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agentbot"} -copilot_unified_name = "Copilot" -copilot_unified_email = "copilot@example.com" ## # RUN POSTPROCESSING @@ -112,25 +107,6 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth :param bots_list: file name of the corresponding bot data :param unify_copilot_users: whether to unify known copilot users into a single user """ - github_user = "GitHub" - github_email = "noreply@github.com" - commit_added_event = "commit_added" - mentioned_event = "mentioned" - subscribed_event = "subscribed" - - """ - Helper function to check whether a (name, e-mail) pair belongs to the author "GitHub ". - There are two options in Codeface how this can happen: - (1) Username is "GitHub" and e-mail address is "noreply@github.com" - (2) Username is "GitHub" and e-mail address has been replaced by Codeface, resulting in "GitHub.noreply@github.com" - - :param name: the name of the author to be checked - :param email: the email address of the author to be checked - :return: whether the given (name, email) pair belongs to the "GitHub " author - """ - def is_github_noreply_author(name, email): - return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) - # Check for all files in the result directory of the project whether they need to be adjusted for filepath, _, filenames in walk(data_path): @@ -139,20 +115,32 @@ def is_github_noreply_author(name, email): if authors_list in filenames: f = path.join(filepath, authors_list) log.info("Remove author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) author_data = csv_writer.read_from_csv(f) author_data_new = [] - + copilot_user_added = False for author in author_data: # keep author entry only if it should not be removed if not is_github_noreply_author(author[1], author[2]): - author_data_new.append(author) + # unify copilot author if desired + if unify_copilot_users and author[1] in known_copilot_users: + if not copilot_user_added: + author[1] = copilot_unified_name + author[2] = copilot_unified_email + copilot_user_added = True + author_data_new.append(author) + else: + author_data_new.append(author) csv_writer.write_to_csv(f, author_data_new) # (2) Remove e-mails from author 'GitHub ' from all emails.list files if emails_list in filenames: f = path.join(filepath, emails_list) log.info("Remove emails from author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) email_data = csv_writer.read_from_csv(f) email_data_new = [] @@ -160,6 +148,10 @@ def is_github_noreply_author(name, email): for email in email_data: # keep author entry only if it should not be removed if not is_github_noreply_author(email[0], email[1]): + # unify copilot users if desired + if unify_copilot_users and email[0] in known_copilot_users: + email[0] = copilot_unified_name + email[1] = copilot_unified_email email_data_new.append(email) else: log.warning("Remove email %s as it was sent by %s <%s>.", email[2], email[0], email[1]) @@ -170,6 +162,8 @@ def is_github_noreply_author(name, email): if commits_list in filenames: f = path.join(filepath, commits_list) log.info("Replace author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) commit_data = csv_writer.read_from_csv(f) for commit in commit_data: @@ -178,6 +172,10 @@ def is_github_noreply_author(name, email): if is_github_noreply_author(commit[5], commit[6]): commit[5] = commit[2] commit[6] = commit[3] + # unify copilot author if desired + if unify_copilot_users and commit[5] in known_copilot_users: + commit[5] = copilot_unified_name + commit[6] = copilot_unified_email csv_writer.write_to_csv(f, commit_data) @@ -186,6 +184,8 @@ def is_github_noreply_author(name, email): if issues_github_list in filenames: f = path.join(filepath, issues_github_list) log.info("Replace author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) issue_data = csv_writer.read_from_csv(f) # read commit data @@ -200,7 +200,13 @@ def is_github_noreply_author(name, email): if unify_copilot_users and event[9] in known_copilot_users: event[9] = copilot_unified_name event[10] = copilot_unified_email - + if event[8] == commit_added_event and event[13][-1:1] in known_copilot_users: + # for commit added events, also unify the referenced author in event info 2 if it is a known copilot user + event[13] = '"' + copilot_unified_name + '"' + elif event[8] in (mentioned_event, subscribed_event) and event[12][-1:1] in known_copilot_users: + # for mentioned/subscribed events, also unify the referenced user in event info 1 and 2 if it is a known copilot user + event[12] = '"' + copilot_unified_name + '"' + event[13] = '"' + copilot_unified_email + '"' # replace author if necessary if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 @@ -380,6 +386,9 @@ def run_postprocessing(conf, resdir, backup_data): if person[4] == issue_event[12] and (quot_m + person[5] + quot_m) == issue_event[13]: issue_event[12] = person[1] issue_event[13] = quot_m + person[2] + quot_m + # replace name in event info 2 if necessary + if person[4] == issue_event[13]: + issue_event[13] = person[1] csv_writer.write_to_csv(f, issue_data) diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py new file mode 100644 index 0000000..20a3aa3 --- /dev/null +++ b/github_user_utils/github_user_utils.py @@ -0,0 +1,54 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Leo Sendelbach +# All Rights Reserved. +""" +This file serves as a collection of global variables and utility functions, which are used throughout the +issue data extraction and post-processing, in particular for the processing of GitHub and Copilot user data. +""" + +## +# GLOBAL VARIABLES +## + +# global variables containing all known copilot users and the name and mail adress copilot users will be assigned +known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agentbot"} +copilot_unified_name = "Copilot" +copilot_unified_email = "copilot@example.com" + +## global variables for the GitHub author +github_user = "GitHub" +github_email = "noreply@github.com" +commit_added_event = "commit_added" +mentioned_event = "mentioned" +subscribed_event = "subscribed" + +## +# UTILITY FUNCTIONS +## + +def is_github_noreply_author(name, email): + """ + Helper function to check whether a (name, e-mail) pair belongs to the author "GitHub ". + There are two options in Codeface how this can happen: + (1) Username is "GitHub" and e-mail address is "noreply@github.com" + (2) Username is "GitHub" and e-mail address has been replaced by Codeface, resulting in "GitHub.noreply@github.com" + + :param name: the name of the author to be checked + :param email: the email address of the author to be checked + :return: whether the given (name, email) pair belongs to the "GitHub " author + """ + + return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) \ No newline at end of file diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index bd1c191..f1ee54c 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -40,6 +40,7 @@ from dateutil import parser as dateparser from csv_writer import csv_writer +from github_user_utils import copilot_unified_name # create logger setup_logging() @@ -57,8 +58,6 @@ # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" -# Copilot username to be assigned in specific copilot events -copilot_username = "Copilot" def run(): # get all needed paths and arguments for the method call. @@ -497,7 +496,7 @@ def merge_issue_events(issue_data, external_connected_events): # if event is a specific copilot event, assign the copilot user data if event["event"] == "copilot_work_started" or event["event"] == "copilot_work_finished": event["user"]["name"] = None - event["user"]["username"] = copilot_username + event["user"]["username"] = copilot_unified_name event["user"]["email"] = "" # if event dismisses a review, we can determine the original state of the corresponding review From 17f7da74225ffcb22c50cbd27d9065139f28a122 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Wed, 18 Feb 2026 14:14:11 +0100 Subject: [PATCH 19/79] Add support for 'known agents' Known agentsc such as 'copilot' or 'claude' can now be read, similar to known bots. They will be flagged as agents during bot processing. Signed-off-by: Leo Sendelbach --- bot_processing/bot_processing.py | 34 ++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 9b18dd4..8c7df78 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -13,6 +13,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright 2021-2022 by Thomas Bock +# Copyright 2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -54,6 +55,7 @@ def run(): # (the known bots file is the file in which known bots have been added manually and project independent) __confdir = os.path.join(args.resdir, os.path.dirname(args.config)) __known_bots_file = os.path.abspath(os.path.join(__confdir, "known_github_bots.list")) + __known_agents_file = os.path.abspath(os.path.join(__confdir, "known_github_agents.list")) # run processing of bot data: # 1) load bot data @@ -61,7 +63,7 @@ def run(): # 2) load user data users = load_user_data(os.path.join(__resdir, "usernames.list")) # 3) update bot data with user data and additionally add known bots if they occur in the project - bots = add_user_data(bots, users, __known_bots_file) + bots = add_user_data(bots, users, __known_bots_file, __known_agents_file) # 4) dump result to disk print_to_disk(bots, __resdir) @@ -113,12 +115,13 @@ def load_user_data(user_data_file): return user_data -def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_reduced): +def check_with_known_bot_or_agent_list(known_bots_file, known_agents_file, bot_data, user_data, bot_data_reduced): """ Check whether there are known bots occurring in the project. If so, add them to the bots list or update the bots list accordingly. :param known_bots_file: the file path to the list of known bot data + :param known_agents_file: the file path to the list of known agent data :param bot_data: the bot data originating from the bot prediction :param user_data: a dictionary from the issue data which maps GitHub usernames to authors :param bot_data_reduced: the bot data after mapping GitHub user names to authors @@ -128,6 +131,7 @@ def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_red # Read the list of known bots known_bots = load_bot_data(known_bots_file, header = False) + known_agents = load_bot_data(known_agents_file, header = False) # Get the GitHub usernames of the bots predicted to be a bot predicted_bots = [bot[0] if len(bot) > 0 else "" for bot in bot_data] @@ -154,11 +158,33 @@ def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_red log.info("Mark user '{}' as bot in the bot data.".format(user_data[bot[0]])) break + for agent in known_agents: + + # (1) check if a known agent occurs in the GitHub issue data but has not been predicted + if agent[0] not in predicted_bots and agent[0] in user_data: + + # add the known agent as a bot to the bots list + additional_agent = dict() + additional_agent["user"] = user_data[agent[0]] + additional_agent["prediction"] = "Agent" + bot_data_reduced.append(additional_agent) + log.info("Add known agent '{}' to bot data.".format(additional_agent["user"])) + + # (2) handle known agents that are already present in the bots list + elif agent[0] in predicted_bots and agent[0] in user_data: + + # make sure that this bot has also been predicited to be an agent + for predicted_bot in bot_data_reduced: + if predicted_bot["user"] == user_data[agent[0]]: + predicted_bot["prediction"] = "Agent" + log.info("Mark user '{}' as agent in the bot data.".format(user_data[agent[0]])) + break + # return the updated bot data return bot_data_reduced -def add_user_data(bot_data, user_data, known_bots_file): +def add_user_data(bot_data, user_data, known_bots_file, known_agents_file): """ Add user data to bot data, i.e., replace username by name and e-mail. In addition, check in the global bots list whether there are authors in the projects which are @@ -202,7 +228,7 @@ def add_user_data(bot_data, user_data, known_bots_file): log.warning("User '{}' in bot data does not occur in GitHub user data. Remove user...".format(user[0])) # check whether known GitHub bots occur in the GitHub issue data and, if so, update the bot data accordingly - bot_data_reduced = check_with_known_bot_list(known_bots_file, bot_data, user_buffer, bot_data_reduced) + bot_data_reduced = check_with_known_bot_or_agent_list(known_bots_file, known_agents_file, bot_data, user_buffer, bot_data_reduced) return bot_data_reduced From 93be3d67ec66fbc6a07fe58c5df50ffcd0932160 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Wed, 18 Feb 2026 14:31:59 +0100 Subject: [PATCH 20/79] Add better bot name variant support Add a helper function for creating bot name variants utilizing either '[bot]' or 'bot' suffix. Also update bot processing to check user buffer for all variants. Signed-off-by: Leo Sendelbach --- .../author_postprocessing.py | 16 +++++++------- bot_processing/bot_processing.py | 8 +++++++ github_user_utils/github_user_utils.py | 21 ++++++++++++++++++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index e908a2c..45ef0e7 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -55,9 +55,9 @@ log = getLogger(__name__) from github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ is_github_noreply_author, github_user, github_email, \ - commit_added_event, mentioned_event, subscribed_event - + commit_added_event, mentioned_event, subscribed_event, generate_botname_variants +known_copilot_users_extended = generate_botname_variants(known_copilot_users) ## # RUN POSTPROCESSING ## @@ -125,7 +125,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth # keep author entry only if it should not be removed if not is_github_noreply_author(author[1], author[2]): # unify copilot author if desired - if unify_copilot_users and author[1] in known_copilot_users: + if unify_copilot_users and author[1] in known_copilot_users_extended: if not copilot_user_added: author[1] = copilot_unified_name author[2] = copilot_unified_email @@ -149,7 +149,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth # keep author entry only if it should not be removed if not is_github_noreply_author(email[0], email[1]): # unify copilot users if desired - if unify_copilot_users and email[0] in known_copilot_users: + if unify_copilot_users and email[0] in known_copilot_users_extended: email[0] = copilot_unified_name email[1] = copilot_unified_email email_data_new.append(email) @@ -173,7 +173,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth commit[5] = commit[2] commit[6] = commit[3] # unify copilot author if desired - if unify_copilot_users and commit[5] in known_copilot_users: + if unify_copilot_users and commit[5] in known_copilot_users_extended: commit[5] = copilot_unified_name commit[6] = copilot_unified_email @@ -197,13 +197,13 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth for event in issue_data: # unify events to use a single copilot user for all events triggered by a known copilot user - if unify_copilot_users and event[9] in known_copilot_users: + if unify_copilot_users and event[9] in known_copilot_users_extended: event[9] = copilot_unified_name event[10] = copilot_unified_email - if event[8] == commit_added_event and event[13][-1:1] in known_copilot_users: + if event[8] == commit_added_event and event[13][-1:1] in known_copilot_users_extended: # for commit added events, also unify the referenced author in event info 2 if it is a known copilot user event[13] = '"' + copilot_unified_name + '"' - elif event[8] in (mentioned_event, subscribed_event) and event[12][-1:1] in known_copilot_users: + elif event[8] in (mentioned_event, subscribed_event) and event[12][-1:1] in known_copilot_users_extended: # for mentioned/subscribed events, also unify the referenced user in event info 1 and 2 if it is a known copilot user event[12] = '"' + copilot_unified_name + '"' event[13] = '"' + copilot_unified_email + '"' diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 8c7df78..5d58eb9 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -224,6 +224,14 @@ def add_user_data(bot_data, user_data, known_bots_file, known_agents_file): bot_reduced["user"] = user_buffer[user[0]] bot_reduced["prediction"] = user[-1] bot_data_reduced.append(bot_reduced) + elif user[0] + "bot" in user_buffer.keys(): + bot_reduced["user"] = user_buffer[user[0] + "bot"] + bot_reduced["prediction"] = user[-1] + bot_data_reduced.append(bot_reduced) + elif user[0] + "[bot]" in user_buffer.keys(): + bot_reduced["user"] = user_buffer[user[0] + "[bot]"] + bot_reduced["prediction"] = user[-1] + bot_data_reduced.append(bot_reduced) else: log.warning("User '{}' in bot data does not occur in GitHub user data. Remove user...".format(user[0])) diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py index 20a3aa3..561a345 100644 --- a/github_user_utils/github_user_utils.py +++ b/github_user_utils/github_user_utils.py @@ -51,4 +51,23 @@ def is_github_noreply_author(name, email): :return: whether the given (name, email) pair belongs to the "GitHub " author """ - return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) \ No newline at end of file + return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) + +def generate_botname_variants(botnames): + """ + Helper function to generate variants of bot names, which are used in the list of + known bots and agents as well as during author postprocessing. + + :param botnames: the list of bot names for which variants should be generated + :return: a set of bot name variants + """ + + botname_variants = set() + for botname in botnames: + botname_variants.add(botname) + if botname.endswith("[bot]"): + botname_variants.add(botname[:-5] + "bot") + elif botname.endswith("bot"): + botname_variants.add(botname[:-3] + "[bot]") + + return botname_variants From 7c436c331adf7c237885b6603217d535a164d89d Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Fri, 20 Feb 2026 17:17:46 +0100 Subject: [PATCH 21/79] Add better bot name handling Add a helper function that given a botname and a list of names, returns which bot name variant is contained in the list (or None). This is used whenever we check if a known bot is in the userdata or has been predicted to be a bot, and means that botnames in the known_bots file do not need to be duplicated for each variant. Also, automatically add all known coplilot users to the known_agents list, and then unify those during author postprocessing. Signed-off-by: Leo Sendelbach --- .../author_postprocessing.py | 13 ++++- bot_processing/bot_processing.py | 55 ++++++++++++++----- github_user_utils/github_user_utils.py | 8 +-- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index 45ef0e7..c19d2bc 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -250,6 +250,9 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth if bots_list in filenames: f = path.join(filepath, bots_list) log.info("Remove author %s <%s> from %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) + copilot_user_added = False bot_data = csv_writer.read_from_csv(f) bot_data_new = [] @@ -257,7 +260,15 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth for entry in bot_data: # keep bot entry only if it should not be removed if not is_github_noreply_author(entry[0], entry[1]): - bot_data_new.append(entry) + # unify copilot users if desired + if unify_copilot_users and entry[0] in known_copilot_users_extended: + if not copilot_user_added: + entry[0] = copilot_unified_name + entry[1] = copilot_unified_email + copilot_user_added = True + bot_data_new.append(entry) + else: + bot_data_new.append(entry) else: log.warning("Remove entry %s <%s> from bots list.", entry[0], entry[1]) diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 5d58eb9..b63f87e 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -28,6 +28,7 @@ from codeface_utils.configuration import Configuration from csv_writer import csv_writer +from github_user_utils import known_copilot_users, generate_botname_variants # create logger setup_logging() @@ -139,25 +140,35 @@ def check_with_known_bot_or_agent_list(known_bots_file, known_agents_file, bot_d for bot in known_bots: # (1) check if a known bot occurs in the GitHub issue data but has not been predicted - if bot[0] not in predicted_bots and bot[0] in user_data: + bot_variation_predicted_bots = containing_bot_variation(bot[0], predicted_bots) + bot_variation_user_data = containing_bot_variation(bot[0], user_data) + if bot_variation_predicted_bots is None and bot_variation_user_data is not None: # add the known bot as a bot to the bots list additional_bot = dict() - additional_bot["user"] = user_data[bot[0]] + additional_bot["user"] = user_data[bot_variation_user_data] additional_bot["prediction"] = "Bot" bot_data_reduced.append(additional_bot) log.info("Add known bot '{}' to bot data.".format(additional_bot["user"])) # (2) handle known bots that are already present in the bots list - elif bot[0] in predicted_bots and bot[0] in user_data: + elif bot_variation_predicted_bots is not None and bot_variation_user_data is not None: # make sure that this bot has also been predicited to be bot for predicted_bot in bot_data_reduced: - if predicted_bot["user"] == user_data[bot[0]]: + if predicted_bot["user"] == user_data[bot_variation_user_data]: predicted_bot["prediction"] = "Bot" - log.info("Mark user '{}' as bot in the bot data.".format(user_data[bot[0]])) + log.info("Mark user '{}' as bot in the bot data.".format(user_data[bot_variation_user_data])) break + # get list of known agents and combine it with the list of known copilot users + copilot_users_variants = generate_botname_variants(known_copilot_users) + # get list of known agent names + known_agents_names = [agent[0] for agent in known_agents] + for copilot_user in copilot_users_variants: + if copilot_user not in known_agents_names: + known_agents.append([copilot_user]) + for agent in known_agents: # (1) check if a known agent occurs in the GitHub issue data but has not been predicted @@ -220,16 +231,9 @@ def add_user_data(bot_data, user_data, known_bots_file, known_agents_file): continue # get user information if available - if user[0] in list(user_buffer.keys()): - bot_reduced["user"] = user_buffer[user[0]] - bot_reduced["prediction"] = user[-1] - bot_data_reduced.append(bot_reduced) - elif user[0] + "bot" in user_buffer.keys(): - bot_reduced["user"] = user_buffer[user[0] + "bot"] - bot_reduced["prediction"] = user[-1] - bot_data_reduced.append(bot_reduced) - elif user[0] + "[bot]" in user_buffer.keys(): - bot_reduced["user"] = user_buffer[user[0] + "[bot]"] + bot_variation = containing_bot_variation(user[0], user_buffer.keys()) + if bot_variation is not None: + bot_reduced["user"] = user_buffer[bot_variation] bot_reduced["prediction"] = user[-1] bot_data_reduced.append(bot_reduced) else: @@ -241,6 +245,27 @@ def add_user_data(bot_data, user_data, known_bots_file, known_agents_file): return bot_data_reduced +def containing_bot_variation(botname, name_list): + """ + Helper function to return the variation of a given bot name that occurs in a list of names. + + :param botname: the bot name for which the variation should be returned + :param name_list: the list of names to be checked for containing the bot name or a variation of it + :return: the variation of the given bot name that occurs in the given list of names, or None if no such variation exists + """ + + if botname in name_list: + return botname + elif botname + "bot" in name_list: + return botname + "bot" + elif botname + "[bot]" in name_list: + return botname + "[bot]" + elif botname.replace("[", "").replace("]", "") in name_list: + return botname.replace("[", "").replace("]", "") + else: + return None + + def print_to_disk(bot_data, results_folder): """ Print bot data to file "bots.list" in result folder. diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py index 561a345..652b63b 100644 --- a/github_user_utils/github_user_utils.py +++ b/github_user_utils/github_user_utils.py @@ -24,7 +24,7 @@ ## # global variables containing all known copilot users and the name and mail adress copilot users will be assigned -known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agentbot"} +known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agent[bot]"} copilot_unified_name = "Copilot" copilot_unified_email = "copilot@example.com" @@ -65,9 +65,7 @@ def generate_botname_variants(botnames): botname_variants = set() for botname in botnames: botname_variants.add(botname) - if botname.endswith("[bot]"): - botname_variants.add(botname[:-5] + "bot") - elif botname.endswith("bot"): - botname_variants.add(botname[:-3] + "[bot]") + botname = botname.replace("[", "").replace("]", "") + botname_variants.add(botname) return botname_variants From f0f95b3d2cf8e88e939349143097e88eabffac6d Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 3 Mar 2026 15:43:11 +0100 Subject: [PATCH 22/79] Add copilot user unification for more events also add agents to bot handling, fix formatting for event_info_2 and subissues also fix a typo where strings would not have their quotes correctly removed Signed-off-by: Leo Sendelbach --- .../author_postprocessing.py | 42 +++++++++++-------- bot_processing/bot_processing.py | 2 +- github_user_utils/__init__.py | 1 + github_user_utils/github_user_utils.py | 7 ++++ issue_processing/issue_processing.py | 8 ++-- 5 files changed, 38 insertions(+), 22 deletions(-) create mode 100644 github_user_utils/__init__.py diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index c19d2bc..cc0b912 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -50,12 +50,15 @@ from codeface_utils.configuration import Configuration from csv_writer import csv_writer +from github_user_utils.github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ + is_github_noreply_author, github_user, github_email, \ + commit_added_event, mentioned_event, subscribed_event, \ + assigned_event, unassigned_event, review_requested_event, \ + review_request_removed_event, generate_botname_variants, quot_m + # create logger setup_logging() log = getLogger(__name__) -from github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ - is_github_noreply_author, github_user, github_email, \ - commit_added_event, mentioned_event, subscribed_event, generate_botname_variants known_copilot_users_extended = generate_botname_variants(known_copilot_users) ## @@ -176,6 +179,9 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth if unify_copilot_users and commit[5] in known_copilot_users_extended: commit[5] = copilot_unified_name commit[6] = copilot_unified_email + if unify_copilot_users and commit[2] in known_copilot_users_extended: + commit[2] = copilot_unified_name + commit[3] = copilot_unified_email csv_writer.write_to_csv(f, commit_data) @@ -194,19 +200,20 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth commit_hash_to_author = {commit[7]: commit[2:4] for commit in commit_data} author_name_to_data = {author[1]: author[1:3] for author in author_data_new} issue_data_new = [] - for event in issue_data: # unify events to use a single copilot user for all events triggered by a known copilot user if unify_copilot_users and event[9] in known_copilot_users_extended: event[9] = copilot_unified_name event[10] = copilot_unified_email - if event[8] == commit_added_event and event[13][-1:1] in known_copilot_users_extended: - # for commit added events, also unify the referenced author in event info 2 if it is a known copilot user - event[13] = '"' + copilot_unified_name + '"' - elif event[8] in (mentioned_event, subscribed_event) and event[12][-1:1] in known_copilot_users_extended: - # for mentioned/subscribed events, also unify the referenced user in event info 1 and 2 if it is a known copilot user - event[12] = '"' + copilot_unified_name + '"' - event[13] = '"' + copilot_unified_email + '"' + if unify_copilot_users and event[8] == commit_added_event and event[13][1:-1] in known_copilot_users_extended: + # for commit added events, also unify the referenced author in event info 2 if it is a known copilot user + event[13] = quot_m + copilot_unified_name + quot_m + elif unify_copilot_users and event[8] in (mentioned_event, subscribed_event, assigned_event, unassigned_event, + review_requested_event, review_request_removed_event) \ + and event[12] in known_copilot_users_extended: + # for mentioned/subscribed events, also unify the referenced user in event info 1 and 2 if it is a known copilot user + event[12] = copilot_unified_name + event[13] = quot_m + copilot_unified_email + quot_m # replace author if necessary if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 @@ -305,9 +312,6 @@ def run_postprocessing(conf, resdir, backup_data): bugs_jira_list = "bugs-jira.list" bots_list = "bots.list" - # When looking at elements originating from json lists, we need to consider quotation marks around the string - quot_m = "\"" - data_path = path.join(resdir, conf["project"], conf["tagging"]) # Correctly replace author 'GitHub ' in the commit data and in "commit_added" events of the @@ -398,8 +402,8 @@ def run_postprocessing(conf, resdir, backup_data): issue_event[12] = person[1] issue_event[13] = quot_m + person[2] + quot_m # replace name in event info 2 if necessary - if person[4] == issue_event[13]: - issue_event[13] = person[1] + if quot_m + person[4] + quot_m == issue_event[13]: + issue_event[13] = quot_m + person[1] + quot_m csv_writer.write_to_csv(f, issue_data) @@ -466,8 +470,12 @@ def run_postprocessing(conf, resdir, backup_data): # the bot is already in the list, check if there are different predictions stored_bot = bot_names_and_emails[(bot[0], bot[1])] if stored_bot[2] != bot[2]: + # if either of the predictions is agent, keep agent + if (stored_bot[2] == "Agent" or bot[2] == "Agent"): + stored_bot[2] = "Agent" + bot_names_and_emails[(bot[0], bot[1])] = stored_bot # if either of the predictions is bot, keep bot - if (stored_bot[2] == "Bot" or bot[2] == "Bot"): + elif (stored_bot[2] == "Bot" or bot[2] == "Bot"): stored_bot[2] = "Bot" bot_names_and_emails[(bot[0], bot[1])] = stored_bot # otherwise, if either of the predictions is human, keep human diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index b63f87e..89bb559 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -28,7 +28,7 @@ from codeface_utils.configuration import Configuration from csv_writer import csv_writer -from github_user_utils import known_copilot_users, generate_botname_variants +from github_user_utils.github_user_utils import known_copilot_users, generate_botname_variants # create logger setup_logging() diff --git a/github_user_utils/__init__.py b/github_user_utils/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/github_user_utils/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py index 652b63b..20fa8d8 100644 --- a/github_user_utils/github_user_utils.py +++ b/github_user_utils/github_user_utils.py @@ -34,6 +34,13 @@ commit_added_event = "commit_added" mentioned_event = "mentioned" subscribed_event = "subscribed" +assigned_event = "assigned" +unassigned_event = "unassigned" +review_requested_event = "review_requested" +review_request_removed_event = "review_request_removed" + +# When looking at elements originating from json lists, we need to consider quotation marks around the string +quot_m = "\"" ## # UTILITY FUNCTIONS diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index f1ee54c..a81de96 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -40,7 +40,7 @@ from dateutil import parser as dateparser from csv_writer import csv_writer -from github_user_utils import copilot_unified_name +from github_user_utils.github_user_utils import copilot_unified_name # create logger setup_logging() @@ -757,9 +757,9 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev event["event_info_1"] = issue["state_new"] # if event is a review comment, it can contain suggestions if "contains_suggestion" in event: - event["event_info_2"] = event["contains_suggestion"] + event["event_info_2"] = str(event["contains_suggestion"]) else: - event["event_info_2"] = False + event["event_info_2"] = str(False) elif event["event"] == "referenced" and event["commit"] is not None: # remove "referenced" events originating from commits @@ -939,7 +939,7 @@ def print_to_disk(issues, results_folder): json.dumps(issue["resolution"]), issue["created_at"], issue["closed_at"], - json.dumps([issue["subIssues"]]), # components + json.dumps(issue["subIssues"]), # components event["event"], event["user"]["name"], event["user"]["email"], From 6ed1def97d10c2c03cb9be57d12549645188d4b5 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 10 Mar 2026 13:58:06 +0100 Subject: [PATCH 23/79] Add reason for conversation locking lock reason is saved in event_info_1 Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index a81de96..7e992ac 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -537,6 +537,10 @@ def merge_issue_events(issue_data, external_connected_events): connected_info["user"] = event["user"] connected_events[event["created_at"]] = connected_info + # if event is a locked event, save the lock reason in event_info_1 + if event["event"] == "locked": + event["event_info_1"] = event["lock_reason"] + # merge events, relatedCommits, relatedIssues and comment lists issue["eventsList"] = issue["commentsList"] + issue["eventsList"] + issue["relatedIssues"] + issue[ "relatedCommits"] + issue["reviewsList"] From 17810bec25b43f30a42f19a011aa9693b40d4304 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 10 Mar 2026 14:09:27 +0100 Subject: [PATCH 24/79] Fix spelling and documentation docstrings should now more accurately reflect parameters and return values Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 7e992ac..858dad1 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -237,7 +237,6 @@ def reformat_issues(issue_data): Re-arrange issue data structure. :param issue_data: the issue data to re-arrange - :return: the re-arranged issue data """ log.info("Re-arranging Github issues...") @@ -302,7 +301,8 @@ def merge_issue_events(issue_data, external_connected_events): All issue events are merged together in the eventsList. This simplifies processing in later steps. :param issue_data: the issue data from which the events shall be merged - :return: the issue data with merged eventsList + :param external_connected_events: a dict to store connected events to external issues + :return: a filtered dict of connected events for future reconstruction """ log.info("Merge issue events ...") @@ -576,17 +576,17 @@ def filter_connected_events(key, value, external_connected_events): # if 2 connected events exist, matching them is trivial if num_issues == 2: return True - occurences = {x: value["issues"].count(x) for x in set(value["issues"])} + occurrences = {x: value["issues"].count(x) for x in set(value["issues"])} # otherwise, if it is an even number, check if it can be easily matched, # meaning that exactly half the events occur in the same issue - if num_issues % 2 == 0 and num_issues/2 in occurences.values(): + if num_issues % 2 == 0 and num_issues/2 in occurrences.values(): # duplicate issue list for matching the issues later value["multi_issues_copy"] = list(value["issues"]) return True # if it is an odd number, check if it can be easily matched # meaning that exactly half (rounded up) the events occur in the same issue - if num_issues % 2 == 1 and (num_issues + 1)/2 in occurences.values(): - for sub_key, sub_value in occurences.iteritems(): + if num_issues % 2 == 1 and (num_issues + 1)/2 in occurrences.values(): + for sub_key, sub_value in occurrences.iteritems(): # then, assign one of them as an external connected event and proceed as in previous case if sub_value == (num_issues + 1)/2: new_entry = dict() @@ -606,7 +606,8 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev Re-format event information dependent on the event type. :param issue_data: the data of all issues that shall be re-formatted - :return: the issue data with updated event information + :param filtered_connected_events: the dict of connected events which can be reconstructed + :param external_connected_events: the dict of connected events to external issues """ log.info("Update event information ...") @@ -646,9 +647,9 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev # and we only have 2 issues in the list, connect to the other issue event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] else: - # and we have more than two issues, count each issue's occurences - occurences = {x: value["issues"].count(x) for x in set(value["issues"])} - if occurences[issue["number"]] == max(occurences.values()): + # and we have more than two issues, count each issue's occurrences + occurrences = {x: value["issues"].count(x) for x in set(value["issues"])} + if occurrences[issue["number"]] == max(occurrences.values()): # if our issue is the most common one, that means it is the common denominator # for all connected events at this time # so this event connects to any other issue @@ -658,7 +659,7 @@ def reformat_events(issue_data, filtered_connected_events, external_connected_ev event["event_info_1"] = number else: # otherwise, connect this event to the common denominator - event["event_info_1"] = max(occurences, key=occurences.get) + event["event_info_1"] = max(occurrences, key=occurrences.get) # as the user dictionary is created, start re-formating the event information of all issues for issue in issue_data: @@ -788,7 +789,6 @@ def insert_user_data(issues, conf, resdir): :param issues: the issues to retrieve user data from :param conf: the project configuration :param resdir: the directory in which the username-to-user-list should be dumped - :return: the updated issue data """ log.info("Syncing users with ID service...") From ca71f40eefe16a450b7b43981ab23e6c20cc203a Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 10 Mar 2026 14:12:43 +0100 Subject: [PATCH 25/79] Remove old state from jira state_updated events For consistency with github events Signed-off-by: Leo Sendelbach --- issue_processing/jira_issue_processing.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index a431ffa..6999764 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -467,21 +467,18 @@ def load_issues_via_api(issues, persons, url, referenced_bys): for change in changelog.histories: # default values for state and resolution - old_state, new_state, old_resolution, new_resolution = "open", "open", "unresolved", "unresolved" + new_state, old_resolution, new_resolution = "open", "unresolved", "unresolved" # all changes in the issue changelog are checked if they contain a useful information for item in change.items: # state_updated event gets created and added to the issue history if item.field == "status": - if item.fromString is not None: - old_state = item.fromString.lower() if item.toString is not None: new_state = item.toString.lower() history = dict() history["event"] = "state_updated" history["event_info_1"] = new_state - history["event_info_2"] = old_state if hasattr(change, "author"): user = create_user(change.author.displayName, change.author.name, "") else: From 40bb0010a97fed74f04f2d5f6620cbf059ede821 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Fri, 13 Mar 2026 11:32:36 +0100 Subject: [PATCH 26/79] Fix jira processing error previously removed event_info_2 for state_updated event, leading to crashes of the issue processing. Now, it instead contains an empty string. Also fix a minor spelling mistake Signed-off-by: --- bot_processing/bot_processing.py | 2 +- issue_processing/jira_issue_processing.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 89bb559..0d0aa37 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -84,7 +84,7 @@ def load_bot_data(bot_file, header = True): # check if file exists and exit early if not if not os.path.exists(bot_file): - log.error("Bot file '{}' does not exist! Exiting early...".format(bot_file)) + log.error("Bot/Agent file '{}' does not exist (can be empty)! Exiting early...".format(bot_file)) sys.exit(-1) bot_data = csv_writer.read_from_csv(bot_file, delimiter=',') diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index 6999764..dd15c78 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -19,7 +19,7 @@ # Copyright 2020-2021 by Thomas Bock # Copyright 2026 by Thomas Bock # Copyright 2023, 2025 by Maximilian Löffler -# Copyright 2025 by Leo Sendelbach +# Copyright 2025-2026 by Leo Sendelbach # All Rights Reserved. """ This file is able to extract Jira issue data from xml files. @@ -479,6 +479,7 @@ def load_issues_via_api(issues, persons, url, referenced_bys): history = dict() history["event"] = "state_updated" history["event_info_1"] = new_state + history["event_info_2"] = "" if hasattr(change, "author"): user = create_user(change.author.displayName, change.author.name, "") else: From 3f0ea9809833fee90e847baccf1d4b2d6ce4c48e Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 12 May 2026 12:45:05 +0200 Subject: [PATCH 27/79] Fix issue with broken commits in merge events event_info_1 should remain empty in that case Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 858dad1..ae54a17 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -512,7 +512,7 @@ def merge_issue_events(issue_data, external_connected_events): event["user"] = event["assigner"] # if event is merged event, save the hash of the merge commit in event_info_1 - if event["event"] == "merged": + if event["event"] == "merged" and not event["commit"] is None: event["event_info_1"] = event["commit"]["hash"] # if event is connected event, create or add to a matching dict entry by matching timestamps, for later reconstruction From d63e68b95ed112cfc69dd425556e6142cf1e6cee Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 26 May 2026 15:52:05 +0200 Subject: [PATCH 28/79] Fix 'null' in relatedIssues, subIssues fields crash These fields are now replaced with empty lists when null Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index ae54a17..8451ec7 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -270,11 +270,11 @@ def reformat_issues(issue_data): issue["reviewsList"] = [] # if an issue has no relatedIssues, an empty List gets created - if "relatedIssues" not in issue: + if issue["relatedIssues"] is None: issue["relatedIssues"] = [] # if an issue has no sub-issue list, an empty List gets created - if "subIssues" not in issue: + if issue["subIssues"] is None: issue["subIssues"] = [] # add "closed_at" information if not present yet From f66c10f4a0ad4fc9622ce87ddd39934f02084871 Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 9 Jun 2026 13:11:25 +0200 Subject: [PATCH 29/79] Fix for python 3 changing .keys() call on maps after rebase onto python 3 branch Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 8451ec7..29aee23 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -517,15 +517,15 @@ def merge_issue_events(issue_data, external_connected_events): # if event is connected event, create or add to a matching dict entry by matching timestamps, for later reconstruction if event["event"] == "connected": - if event["created_at"] in connected_events.keys() and connected_events[event["created_at"]]["user"] == event["user"]: + if event["created_at"] in list(connected_events.keys()) and connected_events[event["created_at"]]["user"] == event["user"]: # if there is already a connected event at this time by this user, add this event to the list connected_events[event["created_at"]]["issues"].append(issue["number"]) - elif subtract_seconds_from_time(event["created_at"], 1) in connected_events.keys() \ + elif subtract_seconds_from_time(event["created_at"], 1) in list(connected_events.keys()) \ and connected_events[subtract_seconds_from_time(event["created_at"], 1)]["user"] == event["user"]: # same as above, but accounting for a possible difference in timestamps of 1 second between matching events connected_events[subtract_seconds_from_time(event["created_at"], 1)]["issues"].append(issue["number"]) event["created_at"] = subtract_seconds_from_time(event["created_at"], 1) - elif subtract_seconds_from_time(event["created_at"], -1) in connected_events.keys() \ + elif subtract_seconds_from_time(event["created_at"], -1) in list(connected_events.keys()) \ and connected_events[subtract_seconds_from_time(event["created_at"], -1)]["user"] == event["user"]: # same as above, with offset calculated in the other direction connected_events[subtract_seconds_from_time(event["created_at"], -1)]["issues"].append(issue["number"]) From fc5d13ad212aff67654fb1629221d9f15b1029cb Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Tue, 9 Jun 2026 15:51:12 +0200 Subject: [PATCH 30/79] Fix iteritems Iteritems() does not work in python3, instrad use items() Signed-off-by: Leo Sendelbach --- github_user_utils/github_user_utils.py | 2 +- issue_processing/issue_processing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py index 20fa8d8..3db2efd 100644 --- a/github_user_utils/github_user_utils.py +++ b/github_user_utils/github_user_utils.py @@ -23,7 +23,7 @@ # GLOBAL VARIABLES ## -# global variables containing all known copilot users and the name and mail adress copilot users will be assigned +# global variables containing all known copilot users and the name and mail address copilot users will be assigned known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agent[bot]"} copilot_unified_name = "Copilot" copilot_unified_email = "copilot@example.com" diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 29aee23..9f69429 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -586,7 +586,7 @@ def filter_connected_events(key, value, external_connected_events): # if it is an odd number, check if it can be easily matched # meaning that exactly half (rounded up) the events occur in the same issue if num_issues % 2 == 1 and (num_issues + 1)/2 in occurrences.values(): - for sub_key, sub_value in occurrences.iteritems(): + for sub_key, sub_value in occurrences.items(): # then, assign one of them as an external connected event and proceed as in previous case if sub_value == (num_issues + 1)/2: new_entry = dict() From aae915b61523aaba36bfed03322d28b5238cabc1 Mon Sep 17 00:00:00 2001 From: Thomas Bock Date: Sat, 13 Jun 2026 17:54:11 +0200 Subject: [PATCH 31/79] Add missing logger imports and add utf-8 header consistently where missing Signed-off-by: Thomas Bock --- anonymization/anonymization.py | 1 + author_postprocessing/author_postprocessing.py | 1 + bot_processing/bot_processing.py | 1 + codeface_utils/configuration.py | 1 + codeface_utils/dbmanager.py | 2 +- codeface_utils/linktype.py | 1 + codeface_utils/util.py | 1 + issue_processing/issue_processing.py | 1 + issue_processing/jira_issue_processing.py | 1 + 9 files changed, 9 insertions(+), 1 deletion(-) diff --git a/anonymization/anonymization.py b/anonymization/anonymization.py index f4bfecb..291a575 100644 --- a/anonymization/anonymization.py +++ b/anonymization/anonymization.py @@ -34,6 +34,7 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer # create logger diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index cc0b912..5840e4e 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -48,6 +48,7 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer from github_user_utils.github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 0d0aa37..c51ed14 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -27,6 +27,7 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer from github_user_utils.github_user_utils import known_copilot_users, generate_botname_variants diff --git a/codeface_utils/configuration.py b/codeface_utils/configuration.py index e4a654a..49d35ab 100644 --- a/codeface_utils/configuration.py +++ b/codeface_utils/configuration.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/dbmanager.py b/codeface_utils/dbmanager.py index aecc172..407a931 100644 --- a/codeface_utils/dbmanager.py +++ b/codeface_utils/dbmanager.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +# coding=utf-8 # This file is part of Codeface. Codeface is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/linktype.py b/codeface_utils/linktype.py index 617d11f..953ce4f 100644 --- a/codeface_utils/linktype.py +++ b/codeface_utils/linktype.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/util.py b/codeface_utils/util.py index 59402d8..80c1b51 100644 --- a/codeface_utils/util.py +++ b/codeface_utils/util.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 9f69429..01a9fab 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -37,6 +37,7 @@ from codeface_utils.cluster.idManager import dbIdManager, csvIdManager from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager +from codeface_utils.util import setup_logging from dateutil import parser as dateparser from csv_writer import csv_writer diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index dd15c78..b308d4b 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -38,6 +38,7 @@ from codeface_utils.cluster.idManager import dbIdManager, csvIdManager from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager +from codeface_utils.util import setup_logging from csv_writer import csv_writer From a045792990a8c849f58797f672970ca819f69594 Mon Sep 17 00:00:00 2001 From: Thomas Bock Date: Sat, 13 Jun 2026 18:04:58 +0200 Subject: [PATCH 32/79] Fix encoding issue in python3 As strings are already utf-8 encoded, don't convert them to utf-8 encoded strings any more. Signed-off-by: Thomas Bock --- codeface_extraction/extractions.py | 8 +++----- codeface_utils/cluster/idManager.py | 3 ++- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/codeface_extraction/extractions.py b/codeface_extraction/extractions.py index 9c636dd..0210b73 100644 --- a/codeface_extraction/extractions.py +++ b/codeface_extraction/extractions.py @@ -15,6 +15,7 @@ # Copyright 2015-2018 by Claus Hunsen # Copyright 2016, 2018-2019 by Thomas Bock # Copyright 2019, 2021 by Thomas Bock +# Copyright 2026 by Thomas Bock # Copyright 2018 by Barbara Eckl # Copyright 2018 by Tina Schuh # Copyright 2025 by Maximilian Löffler @@ -759,18 +760,15 @@ def fix_name_encoding(name): if name is None: return name - # encode utf-8 - name = name.encode('utf-8') - # find out character set of the encoded name - info = decode_header(str(name)) + info = decode_header(name) try: # Apply correct encoding and return unicode string return str(make_header(info)) except UnicodeDecodeError: # Undo utf-8 encoding and return unicode string - return str(name.decode('utf-8')) + return name except LookupError: # Encoding not found, return string as is return name diff --git a/codeface_utils/cluster/idManager.py b/codeface_utils/cluster/idManager.py index 43a4be5..b7c3c0d 100644 --- a/codeface_utils/cluster/idManager.py +++ b/codeface_utils/cluster/idManager.py @@ -14,6 +14,7 @@ # Copyright 2010, 2011 by Wolfgang Mauerer # Copyright 2012, 2013 by Siemens AG, Wolfgang Mauerer # Copyright 2025 by Maximilian Löffler +# Copyright 2026 by Thomas Bock # All Rights Reserved. # # The code in this file originates from: @@ -225,7 +226,7 @@ def getPersonFromDB(self, person_id): log.exception("Could not reach ID service. Is the server running?\n") raise - result = res.read() + result = res.read().decode("utf-8") jsond = json.loads(result)[0] return (jsond) From 14891e0d12febb7338590012fa3cf2b2b9edc4be Mon Sep 17 00:00:00 2001 From: Leo Sendelbach Date: Mon, 22 Jun 2026 13:00:00 +0200 Subject: [PATCH 33/79] Fix users with missing names Previously had 'None' entries for name, now correctly puts username as name Signed-off-by: Leo Sendelbach --- issue_processing/issue_processing.py | 2 +- issue_processing/jira_issue_processing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 01a9fab..89e0b1d 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -819,7 +819,7 @@ def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames= # ensure string representation for name and e-mail address username = str(user["username"]) - name = str(user["name"]) if "name" in user else username + name = str(user["name"]) if user["name"] is not None else username mail = str(user["email"]) # construct string for ID service and send query diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index b308d4b..909eb8a 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -614,7 +614,7 @@ def get_user_string(name, email): def get_id_and_update_user(user, buffer_db_ids=user_id_buffer): # ensure string representation for name and e-mail address - name = str(user["name"]) if "name" in user else str(user["username"]) + name = str(user["name"]) if user["name"] is not None else str(user["username"]) mail = str(user["email"]) # may be empty # construct string for ID service and send query From 9041693c97b378e6ee97327a11d3d7727dcc4cce Mon Sep 17 00:00:00 2001 From: Thomas Bock Date: Mon, 29 Jun 2026 05:15:52 +0200 Subject: [PATCH 34/79] Remove 'None' user from usernames.list Signed-off-by: Thomas Bock --- issue_processing/issue_processing.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 89e0b1d..3b04013 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -907,11 +907,12 @@ def get_user_from_id(idx, buffer_db=user_buffer): lines = [] for username in username_id_buffer: user = get_user_from_id(username_id_buffer[username]) - lines.append(( - username, - user["name"], - user["email"] - )) + if not username == "None": + lines.append(( + username, + user["name"], + user["email"] + )) log.info("Dump username list to file...") username_dump = os.path.join(resdir, "usernames.list") From c01f793b62283fd21acd1b596270c8a4bc973bc8 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 3 Dec 2025 10:36:10 +0100 Subject: [PATCH 35/79] Zulip data extraction file --- api_data_extraction/zulip_data_extraction.py | 150 +++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 api_data_extraction/zulip_data_extraction.py diff --git a/api_data_extraction/zulip_data_extraction.py b/api_data_extraction/zulip_data_extraction.py new file mode 100644 index 0000000..7da1adb --- /dev/null +++ b/api_data_extraction/zulip_data_extraction.py @@ -0,0 +1,150 @@ +import zulip +import json +import time + +# ZULIP_CONFIG_FILE = "/Users/ritikahiremath/Downloads/zuliprc.txt" +ZULIP_CONFIG_FILE = "zuliprc.txt" +client = zulip.Client(config_file=ZULIP_CONFIG_FILE) + + +def safe_get_topics(stream_id): + """ + Fetches all the topics from Zulip rust. + + :params: stream_id : id of the stream. + :return: list of topics in the stream. If no topics exist then returns empty list. + + """ + while True: + resp = client.get_stream_topics(stream_id=stream_id) + if resp["result"] == "success": + return [t["name"] for t in resp["topics"]] + elif resp["result"] == "error" and resp.get("code") == "RATE_LIMIT_HIT": + retry = int(resp.get("retry_after", 5)) + print(f"Rate limit hit, retrying in {retry}s...") + time.sleep(retry) + else: + print("Error fetching topics:", resp) + return [] + + +def topics_extraction(): + """ + Fetches Extract all streams and topics + + :return: returns a dictory of topics in the stream. + """ + streams = client.get_streams()["streams"] + data = {} + + for i, s in enumerate(streams, 1): + stream_name = s["name"] + print(f"[{i}/{len(streams)}] Getting topics for: {stream_name}") + + topics = safe_get_topics(s["stream_id"]) + data[stream_name] = { + "topics": topics + } + + time.sleep(0.5) + + with open("zulip_streams_and_topics.json", "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + print("Saved zulip_streams_and_topics.json") + return data + + +def load_stream_topics(file_path): + """ + Fetches all the streams and topics from the file. + + :params: file_path : path of the file for streams and topics. + :return: returns the content of the file. + """ + with open(file_path, "r", encoding="utf-8") as f: + return json.load(f) + + + +def fetch_all_messages_for_stream(stream_name,topic_name): + """ + Fetches all the messages in a topic through API call. + + :params: topic_name : name of the topic for which all. the messages should be fetched. + :return: returns all the messages from that topic. + """ + + all_messages = [] + anchor = 0 # Start from oldest + + while True: + request = { + "anchor": anchor, + "num_before": 0, + "num_after": 500, # max allowed + "narrow": [ + {"operator": "stream", "operand": stream_name}, + {"operator": "topic", "operand": topic_name} + ] + } + + resp = client.get_messages(request) + msgs = resp["messages"] + + if not msgs: + break + + all_messages.extend(msgs) + + # Move anchor forward for next batch + anchor = msgs[-1]["id"] + 1 + + time.sleep(0.4) + + return all_messages + +def discussion_id_update(msgs): + # Sort by timestamp ascending + msgs = sorted(msgs, key=lambda m: m["timestamp"]) + + for idx, m in enumerate(msgs, start=1): + m["stream_id"] = f'{m["stream_id"]}#{idx}' + + return msgs + + +def messages_extraction_for_each_stream(streams_with_topics): + final_output = [] + + for stream_name, info in streams_with_topics.items(): + topics = info["topics"] + # go topic wise here. fetch all messages for a topic + for topic in topics: + print(f"\n Fetching all messages for stream: {stream_name} and topic: {topic}") + + msgs = fetch_all_messages_for_stream(stream_name,topic) + + msgs = discussion_id_update(msgs) + + for m in msgs: + final_output.append({ + "discussion_id": m["stream_id"], + "discusssion_topic": m["subject"], + "sender_full_name": m["sender_full_name"], + "sender_email": m["sender_email"], + "stream": stream_name, + "content": m["content"], + "timestamp": m["timestamp"] + }) + + # Save everything + output_file = "issues.json" + with open(output_file, "w") as f: + json.dump(final_output, f, indent=2) + + print(f"\n Saved ALL stream messages to: {output_file}") + +if __name__ == "__main__": + streams_and_topics = load_stream_topics("zulip_streams_and_topics.json") + messages_extraction_for_each_stream(streams_and_topics) From edd31d55d0a312a1f6bbc8e6c6c7038504405981 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 3 Dec 2025 10:36:46 +0100 Subject: [PATCH 36/79] Zulip issue processing file created --- issue_processing/zulip_issue_processing.py | 793 +++++++++++++++++++++ 1 file changed, 793 insertions(+) create mode 100644 issue_processing/zulip_issue_processing.py diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py new file mode 100644 index 0000000..a9a22bd --- /dev/null +++ b/issue_processing/zulip_issue_processing.py @@ -0,0 +1,793 @@ +# new zulip issue processing + +""" +This file is able to extract Zulip issue data from json files. +""" + +# import argparse +# import httplib +import json +import os +import sys +# import urllib +from datetime import datetime, timedelta +import hashlib +import base64 + +import operator +from codeface.cli import log +from codeface.cluster.idManager import idManager +from codeface.configuration import Configuration +from codeface.dbmanager import DBManager +from dateutil import parser as dateparser +from datetime import datetime + +from csv_writer import csv_writer + +# datetime format string +datetime_format = "%Y-%m-%d %H:%M:%S" + +def run(): + # get data from zulip api calls . then format it and apply to codeface extraction + # get all needed paths and arguments for the method call. + parser = argparse.ArgumentParser(prog='codeface-extraction-issues-zulip', description='Codeface extraction') + parser.add_argument('-c', '--config', help="Codeface configuration file", default='codeface.conf') + parser.add_argument('-p', '--project', help="Project configuration file", required=True) + parser.add_argument('resdir', help="Directory to store analysis results in") + + # parse arguments + args = parser.parse_args(sys.argv[1:]) + __codeface_conf, __project_conf = map(os.path.abspath, (args.config, args.project)) + + # create configuration + __conf = Configuration.load(__codeface_conf, __project_conf) + + # get source and results folders + __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) + __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) + + # run processing of issue data: + # 1) load the list of issues + issues = load(__srcdir) + # 2) re-format the issues + issues = reformat_issues(issues) + # 3) merges all issue events into one list + issues = merge_issue_events(issues) + # 4) re-format the eventsList of the issues + issues = reformat_events(issues) + # 5) update user data with Codeface database and dump username-to-name/e-mail list + issues = insert_user_data(issues, __conf, __resdir) + # 6) dump result to disk + print_to_disk(issues, __resdir) + + log.info("Zulip issue processing complete!") + + + +def load(source_folder): + """Load issues from disk. + + :param source_folder: the folder where to find 'issues.json' + :return: the loaded issue data + """ + + srcfile = os.path.join(source_folder, "issues.json") + log.devinfo("Loading Github issues from file '{}'...".format(srcfile)) + + # check if file exists and exit early if not + if not os.path.exists(srcfile): + log.error("Zulip issue file '{}' does not exist! Exiting early...".format(srcfile)) + sys.exit(-1) + + with open(srcfile) as issues_file: + issue_data = json.load(issues_file) + + return issue_data + +#UPDATED +def format_time(time): + """ + Format times from different sources to a consistent time format + + :param time: the time that shall be formatted + :return: the formatted time + """ + + # empty time would be formatted to current date + if time == "" or time is None: + return "" + else: + d = datetime.fromtimestamp(time) + return d.strftime(datetime_format) + + +def subtract_seconds_from_time(time, seconds): + """ + Subtract the specified number of seconds from a date string + + :param time: the date string to subtract the specified seconds from + :param seconds: the number of seconds to subtract from the date string + :return: the date string after subtracting the specified number of seconds + """ + + new_time = datetime.strptime(time, datetime_format) - timedelta(seconds = seconds) + return new_time.strftime(datetime_format) + + +def create_user(name, username, email): + """ + Creates a user object with all needed information + + :param name: the name the user shall have + :param username: the username the user shall have + :param email: the email the user shall have + :return: the created user object + """ + + if name is None: + name = "" + if username is None: + username = "" + if email is None: + email = "" + + user = dict() + user["name"] = name + user["username"] = username + user["email"] = email + + return user + + +def create_deleted_user(): + """ + Creates a user object for a deleted user (ghost user) + + :return: the created user object for a deleted user + """ + + return create_user("Deleted user", "ghost", "ghost@github.com") + + +def lookup_user(user_dict, user): + """ + Alters a user object in the case that name or email are missing by the corresponding name and email + from a given user dictionary + + :param user_dict: the user dictionary + :param user: the user object to lookup in the dictionary + :return: the altered user object in case of a lookup + or the unaltered user object otherwise + """ + + # if user is None, replace it by a deleted user + if user is None: + user = create_deleted_user() + + if (user["name"] == "" or user["name"] is None or + user["email"] is None or user["email"] == ""): + + # lookup user only if username is not None and not empty + if not user["username"] is None and not user["username"] == "": + user = user_dict[user["username"]] + + return user + + +def update_user_dict(user_dict, user): + """ + Adds or updates users to merge GitHub usernames and names and e-mail addresses originating from the git repository + + :param user_dict: the user dictionary + :param user: the user object to add to or update in the dictionary + :return: the updated user dictionary + """ + + # if the given user is None, use the deleted user instead + if user is None: + user = create_deleted_user() + + if not user["username"] in user_dict.keys(): + if not user["username"] is None and not user["username"] == "": + user_dict[user["username"]] = user + else: + user_in_dict = user_dict[user["username"]] + if user_in_dict["name"] is None or user_in_dict["name"] == "": + user_in_dict["name"] = user["name"] + if user_in_dict["email"] is None or user_in_dict["email"] == "": + user_in_dict["email"] = user["email"] + user_dict[user["username"]] = user_in_dict + + return user_dict + + +def reformat_issues(issue_data): + """ + Re-arrange issue data structure. + + :param issue_data: the issue data to re-arrange + :return: the re-arranged issue data + """ + + log.devinfo("Re-arranging Github issues...") + + # re-process all issues + for issue in issue_data: + + # empty container for issue types + issue["type"] = [] + + # empty container for issue resolutions + issue["resolution"] = [] + + # if an issue has no eventsList, an empty List gets created + if issue["eventsList"] is None: + issue["eventsList"] = [] + + # if an issue has no commentsList, an empty List gets created + if issue["commentsList"] is None: + issue["commentsList"] = [] + + # if an issue has no relatedCommits, an empty List gets created + if issue["relatedCommits"] is None: + issue["relatedCommits"] = [] + + # if an issue has no reviewsList, an empty Listgets created + if issue["reviewsList"] is None: + issue["reviewsList"] = [] + + # if an issue has no relatedIssues, an empty List gets created + if "relatedIssues" not in issue: + issue["relatedIssues"] = [] + + # add "closed_at" information if not present yet + # if issue["closed_at"] is None: + # issue["closed_at"] = "" + + # parses the creation time in the correct format + issue["created_at"] = format_time(issue["created_at"]) + + # parses the close time in the correct format + issue["closed_at"] = format_time(issue["closed_at"]) + + # checks if the issue is a pull-request or a normal issue and adapts the type + issue["type"].append("issue") + + return issue_data + +# TO DO: is this needed? +def merge_issue_events(issue_data): + """ + All issue events are merged together in the eventsList. This simplifies processing in later steps. + + :param issue_data: the issue data from which the events shall be merged + :return: the issue data with merged eventsList + """ + + log.info("Merge issue events ...") + + issue_data_to_update = dict() + + for issue in issue_data: + + # temporary container for references + comments = dict() + + # adds creation event to eventsList + created_event = dict() + created_event["user"] = issue["user"] + created_event["created_at"] = issue["created_at"] + created_event["event"] = "created" + created_event["event_info_1"] = "open" + created_event["event_info_2"] = [] + issue["eventsList"].append(created_event) + issue["state_new"] = "open" + + # adds commented event for the creation-event comment to the commentsList + creationComment = dict() + creationComment["event"] = "commented" + creationComment["user"] = issue["user"] + creationComment["referenced_at"] = issue["created_at"] + creationComment["ref_target"] = "" + creationComment["event_info_1"] = "" + creationComment["event_info_2"] = "" + + issue["commentsList"].append(creationComment) + + # the format of every related issue is adjusted to the event format + for rel_issue in issue["relatedIssues"]: + rel_issue["created_at"] = format_time(rel_issue["referenced_at"]) + rel_issue["event"] = "add_link" + rel_issue["event_info_1"] = rel_issue["number"] + rel_issue["event_info_2"] = "issue" + rel_issue["ref_target"] = "" + + # the related issues states that a user has add a link to another issue within the issue of interest, + # now we add an event for the referenced issue which states that it was referenced + referenced_issue_event = dict() + referenced_issue_event["created_at"] = format_time(rel_issue["referenced_at"]) + referenced_issue_event["event"] = "referenced_by" + referenced_issue_event["user"] = rel_issue["user"] + referenced_issue_event["event_info_1"] = issue["number"] + referenced_issue_event["event_info_2"] = "issue" + referenced_issue_event["ref_target"] = "" + + # as we cannot update the referenced issue during iterating over all issues, we need to save the + # referenced_by event for the referenced issue temporarily + if rel_issue["number"] in issue_data_to_update.keys(): + issue_data_to_update[rel_issue["number"]]["eventsList"].append(referenced_issue_event) + else: + ref = dict() + ref["number"] = rel_issue["number"] + ref["eventsList"] = list() + ref["eventsList"].append(referenced_issue_event) + issue_data_to_update[rel_issue["number"]] = ref + + # the format of every related commit is adjusted to the event format + for rel_commit in issue["relatedCommits"]: + + rel_commit["created_at"] = format_time(rel_commit["referenced_at"]) + rel_commit["event_info_1"] = rel_commit["commit"]["hash"] + rel_commit["event_info_2"] = "" + rel_commit["ref_target"] = "" + + # if the related commit is not of type "commit" but "commitAddedToPullRequest", + # it is a commit which was added to the pull request + if rel_commit["type"] == "commitAddedToPullRequest": + rel_commit["event"] = "commit_added" + + # if the related commit was mentioned in an issue comment: + elif rel_commit["type"] == "commitMentionedInIssue": + rel_commit["event"] = "add_link" + rel_commit["event_info_2"] = "commit" + + # else it is a commit which references issue/pull-request + else: + rel_commit["event"] = "referenced_by" + rel_commit["event_info_2"] = "commit" + + # the format of every comment is adjusted to the event format + for comment in issue["commentsList"]: + comment["event"] = "commented" + comment["ref_target"] = "" + comment["created_at"] = format_time(comment["referenced_at"]) + if "event_info_1" not in comment: + comment["event_info_1"] = "" + if "event_info_2" not in comment: + comment["event_info_2"] = "" + + # cache comment by date to resolve/re-arrange references later + comments[comment["created_at"]] = comment + + # the format of every review and their comments is adjusted to the event format + for review in issue["reviewsList"]: + review["event"] = "reviewed" + review["created_at"] = format_time(review["submitted_at"]) + review["event_info_1"] = review["state"].lower() + review["event_info_2"] = "" + review["ref_target"] = "" + + if review["hasReviewInitialComment"]: + initialComment = dict() + initialComment["event"] = "commented" + initialComment["user"] = review["user"] + initialComment["ref_target"] = "" + initialComment["created_at"] = format_time(review["submitted_at"]) + initialComment["event_info_1"] = "" + initialComment["event_info_2"] = "" + + issue["commentsList"].append(initialComment) + + # cache comment by date to resolve/re-arrange references later + comments[initialComment["created_at"]] = initialComment + + for reviewComment in review["reviewComments"]: + reviewComment["event"] = "commented" + reviewComment["created_at"] = format_time(reviewComment["referenced_at"]) + reviewComment["ref_target"] = "" + reviewComment["event_info_1"] = "" + reviewComment["event_info_2"] = "" + + # cache comment by date to resolve/re-arrange references later + comments[reviewComment["created_at"]] = reviewComment + + issue["commentsList"].append(reviewComment) + + # add dismissal comments to the list of comments + for event in issue["eventsList"]: + + if (event["event"] == "review_dismissed" and not event["dismissalMessage"] is None + and not event["dismissalMessage"] == ""): + dismissalComment = dict() + dismissalComment["event"] = "commented" + dismissalComment["user"] = event["user"] + dismissalComment["created_at"] = format_time(event["created_at"]) + dismissalComment["ref_target"] = "" + dismissalComment["event_info_1"] = "" + dismissalComment["event_info_2"] = "" + + # cache comment by date to resolve/re-arrange references later + comments[dismissalComment["created_at"]] = dismissalComment + + issue["commentsList"].append(dismissalComment) + + # the format of every event is adjusted + for event in issue["eventsList"]: + event["ref_target"] = "" + event["created_at"] = format_time(event["created_at"]) + if "event_info_1" not in event: + event["event_info_1"] = "" + if "event_info_2" not in event: + event["event_info_2"] = "" + + # if event collides with a comment + if event["created_at"] in comments: + comment = comments[event["created_at"]] + # if someone gets mentioned or subscribed by someone else in a comment, + # re-write the reference + if (event["event"] == "mentioned" or event["event"] == "subscribed") and \ + comment["event"] == "commented": + event["ref_target"] = event["user"] + event["user"] = comment["user"] + elif subtract_seconds_from_time(event["created_at"], 1) in comments: + comment = comments[subtract_seconds_from_time(event["created_at"], 1)] + # if someone gets mentioned or subscribed by someone else in a comment, + # re-write the reference + if (event["event"] == "mentioned" or event["event"] == "subscribed") and \ + comment["event"] == "commented": + event["ref_target"] = event["user"] + event["user"] = comment["user"] + + # if event is a referenced commit, we can update the user information + if event["event"] == "referenced" and event["commit"] is not None: + if (event["user"] is None): + event["user"] = dict() + event["user"]["name"] = event["commit"]["author"]["name"] # author or committer? + event["user"]["email"] = event["commit"]["author"]["email"] + event["user"]["username"] = event["commit"]["author"]["username"] + + # if event is a review request, we can update the ref target with the requested reviewer + if event["event"] == "review_requested" or event["event"] == "review_request_removed": + event["ref_target"] = event["requestedReviewer"] + + # if event dismisses a review, we can determine the original state of the corresponding review + if event["event"] == "review_dismissed": + for review in issue["reviewsList"]: + if review["reviewId"] == event["reviewId"]: + review["state"] = review["event_info_1"] = event["state"] + review["event_info_2"] = "later_dismissed" + + # if event is assign event, we have set the user to the assigner and the ref target to the assignee + if event["event"] == "assigned" or event["event"] == "unassigned": + event["ref_target"] = event["user"] + event["user"] = event["assigner"] + + # merge events, relatedCommits, relatedIssues and comment lists + issue["eventsList"] = issue["commentsList"] + issue["eventsList"] + issue["relatedIssues"] + issue[ + "relatedCommits"] + issue["reviewsList"] + + # remove events without user + issue["eventsList"] = [event for event in issue["eventsList"] if + not (event["user"] is None or event["ref_target"] is None)] + + # sorts eventsList by time + issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) + + # updates all the issues by the temporarily stored referenced_by events + for key, value in issue_data_to_update.iteritems(): + for issue in issue_data: + if issue["number"] == value["number"]: + issue["eventsList"] = issue["eventsList"] + value["eventsList"] + + return issue_data + + +def reformat_events(issue_data): + """ + Re-format event information dependent on the event type. + + :param issue_data: the data of all issues that shall be re-formatted + :return: the issue data with updated event information + """ + + log.info("Update event information ...") + + users = dict() + + # create a dictionary of users to merge GitHub usernames and names and e-mails originating from the git repository + for issue in issue_data: + + for event in issue["eventsList"]: + + # 1) add or update users which are authors of commits + # (committers of commits are usually the actor of the current event and will be dealt with in part 2 below) + if (event["event"] == "commit_added" or (event["event"] == "add_link" and event["event_info_2"] == "commit") + or (event["event"] == "referenced_by" and event["event_info_2"] == "commit")): + users = update_user_dict(users, event["commit"]["author"]) + + # 2) add or update users which are actor of the current event + users = update_user_dict(users, event["user"]) + + # 3) add or update users which are ref_target of the current event + if not event["ref_target"] is None and not event["ref_target"] == "": + users = update_user_dict(users, event["ref_target"]) + + # as the user dictionary is created, start re-formating the event information of all issues + for issue in issue_data: + + events_to_remove = list() + + # re-format information of every event in the eventsList of an issue + for event in issue["eventsList"]: + + # lookup user in dictionary + event["user"] = lookup_user(users, event["user"]) + if (event["ref_target"] != ""): + event["ref_target"] = lookup_user(users, event["ref_target"]) + + + if event["event"] == "closed": + event["event"] = "state_updated" + event["event_info_1"] = "closed" # new state + event["event_info_2"] = "open" # old state + issue["state_new"] = "closed" + + elif event["event"] == "reopened": + event["event"] = "state_updated" + event["event_info_1"] = "open" # new state + event["event_info_2"] = "closed" # old state + issue["state_new"] = "reopened" + + elif event["event"] == "labeled": + label = event["label"]["name"].lower() + event["event_info_1"] = label + + # if the label is in this list, it also is a type of the issue + if label in known_types: + issue["type"].append(str(label)) + + # creates an event for type updates and adds it to the eventsList + type_event = dict() + type_event["user"] = event["user"] + type_event["created_at"] = event["created_at"] + type_event["event"] = "type_updated" + type_event["event_info_1"] = label + type_event["event_info_2"] = "" + type_event["ref_target"] = "" + issue["eventsList"].append(type_event) + + # if the label is in this list, it also is a resolution of the issue + elif label in known_resolutions: + issue["resolution"].append(str(label)) + + # creates an event for resolution updates and adds it to the eventsList + resolution_event = dict() + resolution_event["user"] = event["user"] + resolution_event["created_at"] = event["created_at"] + resolution_event["event"] = "resolution_updated" + resolution_event["event_info_1"] = label + resolution_event["event_info_2"] = "" + resolution_event["ref_target"] = "" + issue["eventsList"].append(resolution_event) + + elif event["event"] == "unlabeled": + label = event["label"]["name"].lower() + event["event_info_1"] = label + + # if the label is in this list, it also is a type of the issue + if label in known_types: + if label in issue["type"]: + issue["type"].remove(str(label)) + + # creates an event for type updates and adds it to the eventsList + type_event = dict() + type_event["user"] = event["user"] + type_event["created_at"] = event["created_at"] + type_event["event"] = "type_updated" + type_event["event_info_1"] = "" + type_event["event_info_2"] = label + type_event["ref_target"] = "" + issue["eventsList"].append(type_event) + + # if the label is in this list, it also is a resolution of the issue + elif label in known_resolutions: + if label in issue["resolution"]: + issue["resolution"].remove(str(label)) + + # creates an event for resolution updates and adds it to the eventsList + resolution_event = dict() + resolution_event["user"] = event["user"] + resolution_event["created_at"] = event["created_at"] + resolution_event["event"] = "resolution_updated" + resolution_event["event_info_1"] = "" + resolution_event["event_info_2"] = label + resolution_event["ref_target"] = "" + issue["eventsList"].append(resolution_event) + + elif event["event"] == "commented": + # "state_new" and "resolution" of the issue give the information about the state and the resolution of + # the issue when the comment was written, because the eventsList is sorted by time + event["event_info_1"] = issue["state_new"] + event["event_info_2"] = issue["resolution"] + + elif event["event"] == "referenced" and not event["commit"] is None: + # remove "referenced" events originating from commits + # as they are handled as referenced commit + events_to_remove.append(event) + + # sorts eventsList by time again + issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) + + # remove unwanted events + for event_to_remove in events_to_remove: + issue["eventsList"].remove(event_to_remove) + + return issue_data + + +def insert_user_data(issues, conf, resdir): + """ + Insert user data into database and update issue data. + In addition, dump username-to-user list to file. + + :param issues: the issues to retrieve user data from + :param conf: the project configuration + :param resdir: the directory in which the username-to-user-list should be dumped + :return: the updated issue data + """ + + log.info("Syncing users with ID service...") + + # create buffer for users (key: user id) + user_buffer = dict() + # create buffer for user ids (key: user string) + user_id_buffer = dict() + # create buffer for usernames (key: username) + username_id_buffer = dict() + # open database connection + dbm = DBManager(conf) + # open ID-service connection + idservice = idManager(dbm, conf) + + def get_user_string(name, email): + if not email or email is None: + return "{name}".format(name=name) + # return "{name} <{name}@default.com>".format(name=name) # for debugging only + else: + return "{name} <{email}>".format(name=name, email=email) + + def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames=username_id_buffer): + username = unicode(user["username"]).encode("utf-8") + + # fix encoding for name and e-mail address + if user["name"] is not None: + name = unicode(user["name"]).encode("utf-8") + else: + name = username + mail = unicode(user["email"]).encode("utf-8") + # construct string for ID service and send query + user_string = get_user_string(name, mail) + + # check buffer to reduce amount of DB queries + if user_string in buffer_db_ids: + log.devinfo("Returning person id for user '{}' from buffer.".format(user_string)) + if username is not None: + buffer_usernames[username] = buffer_db_ids[user_string] + return buffer_db_ids[user_string] + + # get person information from ID service + log.devinfo("Passing user '{}' to ID service.".format(user_string)) + idx = idservice.getPersonID(user_string) + + # add user information to buffer + # user_string = get_user_string(user["name"], user["email"]) # update for + buffer_db_ids[user_string] = idx + + # add id to username buffer + if username is not None: + buffer_usernames[username] = idx + + return idx + + def get_user_from_id(idx, buffer_db=user_buffer): + + # check whether user information is in buffer to reduce amount of DB queries + if idx in buffer_db: + log.devinfo("Returning user '{}' from buffer.".format(idx)) + return buffer_db[idx] + + # get person information from ID service + log.devinfo("Passing user id '{}' to ID service.".format(idx)) + person = idservice.getPersonFromDB(idx) + user = dict() + user["email"] = person["email1"] # column "email1" + user["name"] = person["name"] # column "name" + user["id"] = person["id"] # column "id" + + # add user information to buffer + buffer_db[idx] = user + + return user + + + # check and update database for all occurring users + for issue in issues: + # check database for issue author + issue["user"] = get_id_and_update_user(issue["user"]) + + # check database for event authors + for event in issue["eventsList"]: + event["user"] = get_id_and_update_user(event["user"]) + + # check database for the reference-target user if needed + if event["ref_target"] != "": + event["ref_target"] = get_id_and_update_user(event["ref_target"]) + + # get all users after database updates having been performed + for issue in issues: + # get issue author + issue["user"] = get_user_from_id(issue["user"]) + + # get event authors + for event in issue["eventsList"]: + event["user"] = get_user_from_id(event["user"]) + + # get the reference-target user if needed + if event["ref_target"] != "": + event["ref_target"] = get_user_from_id(event["ref_target"]) + event["event_info_1"] = event["ref_target"]["name"] + event["event_info_2"] = event["ref_target"]["email"] + + # dump username, name, and e-mail to file + lines = [] + for username in username_id_buffer: + user = get_user_from_id(username_id_buffer[username]) + lines.append(( + username, + user["name"], + user["email"] + )) + + log.info("Dump username list to file...") + username_dump = os.path.join(resdir, "usernames.list") + csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) + + return issues + + +def print_to_disk(issues, results_folder): + """ + Print issues to file "issues-github.list" in the results folder. + + :param issues: the issues to dump + :param results_folder: the folder where to place "issues-github.list" output file + """ + + # construct path to output file + output_file = os.path.join(results_folder, "issues-github.list") + log.info("Dumping output in file '{}'...".format(output_file)) + + # construct lines of output + lines = [] + for issue in issues: + for event in issue["eventsList"]: + lines.append(( + issue["number"], + issue["title"], + json.dumps(issue["type"]), + issue["state_new"], + json.dumps(issue["resolution"]), + issue["created_at"], + issue["closed_at"], + json.dumps([]), # components + event["event"], + event["user"]["name"], + event["user"]["email"], + event["created_at"], + event["event_info_1"], + json.dumps(event["event_info_2"]) + )) + + # write to output file + csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) + From af69c15a7b8b357424e1aa2a112aa17bb0422b2d Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 3 Dec 2025 10:37:20 +0100 Subject: [PATCH 37/79] run zulip issues file created --- run-zulip-issues.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 run-zulip-issues.py diff --git a/run-zulip-issues.py b/run-zulip-issues.py new file mode 100644 index 0000000..d9a44c1 --- /dev/null +++ b/run-zulip-issues.py @@ -0,0 +1,3 @@ +import issue_processing.zulip_issue_processing as zulip_issues + +zulip_issues.run() From 6a1deb2576617bf5c8395fcf83e6afa8e9d8032b Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 8 Dec 2025 12:11:28 +0100 Subject: [PATCH 38/79] zulip_isse_processing updated --- .DS_Store | Bin 0 -> 6148 bytes .gitignore | 1 + api_data_extraction/zulip_data_extraction.py | 16 +- issue_processing/trial.py | 31 ++ issue_processing/zulip_issue_processing.py | 455 ++++--------------- 5 files changed, 115 insertions(+), 388 deletions(-) create mode 100644 .DS_Store create mode 100644 issue_processing/trial.py diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..31e9fb2ab626563d82e455aa0db2cbe3420f0d1a GIT binary patch literal 6148 zcmeHKQA^xF5S~5HnnQ$u6?_czRq)hek-mh}{((m7Ln~==i8eIN9!aiOI1cjM{-OSo z{vN;C-4o?nsC_6RGso;VnVre*eiL?YiAas6*#VJHL>`o}H$d|n;eOT?={e6jP?>jR zlu!$eDPM^e$75uGYqyPxO{hTC?$+-cDp{jv3@k`fl{Mos)w<+mY{RV)p3{{H$xX>u zQqoy7NlTp~<7YZ4i>4h8f2rPDfBo5px8ZGim(eFZi`t}}Hlw6|&rjzVB3;@^`mV@l zvH$#3S8Y;Md2JGkEJw(Pi=xW(Y^0}{o5_uAhd1yBvA;W?zd9N21;1&vT`>RWk$#Q z0~1avv?#4GAPn4RV9h@IeEz@r^ZoyRk~|3m!oZVaK=sd}vk{i$&eoO1@mU){KSEhJ nuCsWX0>d1|h~=Yr7it9lfDK^iu(JpcME(dE8l({h{wo7FyMA?* literal 0 HcmV?d00001 diff --git a/.gitignore b/.gitignore index db79ec1..792941e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.Rhistory .idea/ *.pyc +.DS_Store diff --git a/api_data_extraction/zulip_data_extraction.py b/api_data_extraction/zulip_data_extraction.py index 7da1adb..ac2ce9d 100644 --- a/api_data_extraction/zulip_data_extraction.py +++ b/api_data_extraction/zulip_data_extraction.py @@ -104,14 +104,6 @@ def fetch_all_messages_for_stream(stream_name,topic_name): return all_messages -def discussion_id_update(msgs): - # Sort by timestamp ascending - msgs = sorted(msgs, key=lambda m: m["timestamp"]) - - for idx, m in enumerate(msgs, start=1): - m["stream_id"] = f'{m["stream_id"]}#{idx}' - - return msgs def messages_extraction_for_each_stream(streams_with_topics): @@ -119,18 +111,16 @@ def messages_extraction_for_each_stream(streams_with_topics): for stream_name, info in streams_with_topics.items(): topics = info["topics"] - # go topic wise here. fetch all messages for a topic + for topic in topics: print(f"\n Fetching all messages for stream: {stream_name} and topic: {topic}") msgs = fetch_all_messages_for_stream(stream_name,topic) - - msgs = discussion_id_update(msgs) for m in msgs: final_output.append({ "discussion_id": m["stream_id"], - "discusssion_topic": m["subject"], + "discussion_topic": m["subject"], "sender_full_name": m["sender_full_name"], "sender_email": m["sender_email"], "stream": stream_name, @@ -146,5 +136,5 @@ def messages_extraction_for_each_stream(streams_with_topics): print(f"\n Saved ALL stream messages to: {output_file}") if __name__ == "__main__": - streams_and_topics = load_stream_topics("zulip_streams_and_topics.json") + streams_and_topics = topics_extraction() messages_extraction_for_each_stream(streams_and_topics) diff --git a/issue_processing/trial.py b/issue_processing/trial.py new file mode 100644 index 0000000..f4ad744 --- /dev/null +++ b/issue_processing/trial.py @@ -0,0 +1,31 @@ +import json + +def discussion_id_update(issue_data): + """ + """ + # Group timestamps by discussion_topic + grouped = {} + + for item in issue_data: + topic = item["discussion_topic"] + if topic not in grouped: + grouped[topic] = [] + grouped[topic].append(item) + + # Process each topic group + for topic, messages in grouped.items(): + + # 1. Sort messages by timestamp ASC + messages.sort(key=lambda m: m["timestamp"]) + + # 2. Add sequential discussion_id suffix (#1, #2, #3...) + for idx, msg in enumerate(messages, start=1): + msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' + + return issue_data + + +with open("/Users/ritikahiremath/Desktop/extraction/issue_processing/_issues/_issues.json", "r") as f: + issue_data = json.load(f) +discussion_id_update(issue_data) +print(issue_data) \ No newline at end of file diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index a9a22bd..645cef4 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -49,12 +49,10 @@ def run(): # run processing of issue data: # 1) load the list of issues issues = load(__srcdir) - # 2) re-format the issues + # 2) update missing colums + issues = update(issues) + # 3) re-format the issues issues = reformat_issues(issues) - # 3) merges all issue events into one list - issues = merge_issue_events(issues) - # 4) re-format the eventsList of the issues - issues = reformat_events(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk @@ -200,6 +198,74 @@ def update_user_dict(user_dict, user): return user_dict +def discussion_id_update(issue_data): + """ + :param issue_data: + """ + # Group timestamps by discussion_topic + grouped = {} + + for item in issue_data: + topic = item["discusssion_topic"] + if topic not in grouped: + grouped[topic] = [] + grouped[topic].append(item) + + # Process each topic group + for topic, messages in grouped.items(): + + # 1. Sort messages by timestamp ASC + messages.sort(key=lambda m: m["timestamp"]) + + # 2. Add sequential discussion_id suffix (#1, #2, #3...) + for idx, msg in enumerate(messages, start=1): + msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' + + return issue_data + +def discussion_begin_end_add(issue_data): + """ + :param issue_data: the issue data where discussion_begin and discussion_end time is to be added + :return: the updated data + """ + # Group timestamps by discussion_topic + discussion_topics = {} + + for item in issue_data: + d_topic = item["discussion_topic"] + ts = item["timestamp"] + + if d_topic not in discussion_topics: + discussion_topics[d_topic] = [] + discussion_topics[d_topic].append(ts) + + # Compute begin and end for each discussion + discussion_bounds = { + d_topic: { + "discussion_begin": min(times), + "discussion_end": max(times) + } + for d_topic, times in discussion_topics.items() + } + + # Add begin/end back to each entry + for item in issue_data: + d_topic = item["discussion_topic"] + item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] + item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] + + return issue_data + +def update(issue_data): + """ + updates values in the issue data structure as per requirement. + :params: issue_data: the issue data to be updated.x + :return: returns the issue data. + """ + issue_data = discussion_id_update(issue_data) + issue_data = discussion_begin_end_add(issue_data) + + return issue_data def reformat_issues(issue_data): """ @@ -220,6 +286,7 @@ def reformat_issues(issue_data): # empty container for issue resolutions issue["resolution"] = [] + # TO DO: Are these column names needed? though they will be empty # if an issue has no eventsList, an empty List gets created if issue["eventsList"] is None: issue["eventsList"] = [] @@ -245,384 +312,22 @@ def reformat_issues(issue_data): # issue["closed_at"] = "" # parses the creation time in the correct format - issue["created_at"] = format_time(issue["created_at"]) + # issue["created_at"] = format_time(issue["created_at"]) # parses the close time in the correct format - issue["closed_at"] = format_time(issue["closed_at"]) - - # checks if the issue is a pull-request or a normal issue and adapts the type - issue["type"].append("issue") - - return issue_data - -# TO DO: is this needed? -def merge_issue_events(issue_data): - """ - All issue events are merged together in the eventsList. This simplifies processing in later steps. - - :param issue_data: the issue data from which the events shall be merged - :return: the issue data with merged eventsList - """ - - log.info("Merge issue events ...") - - issue_data_to_update = dict() + # issue["closed_at"] = format_time(issue["closed_at"]) - for issue in issue_data: - - # temporary container for references - comments = dict() - - # adds creation event to eventsList - created_event = dict() - created_event["user"] = issue["user"] - created_event["created_at"] = issue["created_at"] - created_event["event"] = "created" - created_event["event_info_1"] = "open" - created_event["event_info_2"] = [] - issue["eventsList"].append(created_event) - issue["state_new"] = "open" - - # adds commented event for the creation-event comment to the commentsList - creationComment = dict() - creationComment["event"] = "commented" - creationComment["user"] = issue["user"] - creationComment["referenced_at"] = issue["created_at"] - creationComment["ref_target"] = "" - creationComment["event_info_1"] = "" - creationComment["event_info_2"] = "" - - issue["commentsList"].append(creationComment) - - # the format of every related issue is adjusted to the event format - for rel_issue in issue["relatedIssues"]: - rel_issue["created_at"] = format_time(rel_issue["referenced_at"]) - rel_issue["event"] = "add_link" - rel_issue["event_info_1"] = rel_issue["number"] - rel_issue["event_info_2"] = "issue" - rel_issue["ref_target"] = "" - - # the related issues states that a user has add a link to another issue within the issue of interest, - # now we add an event for the referenced issue which states that it was referenced - referenced_issue_event = dict() - referenced_issue_event["created_at"] = format_time(rel_issue["referenced_at"]) - referenced_issue_event["event"] = "referenced_by" - referenced_issue_event["user"] = rel_issue["user"] - referenced_issue_event["event_info_1"] = issue["number"] - referenced_issue_event["event_info_2"] = "issue" - referenced_issue_event["ref_target"] = "" - - # as we cannot update the referenced issue during iterating over all issues, we need to save the - # referenced_by event for the referenced issue temporarily - if rel_issue["number"] in issue_data_to_update.keys(): - issue_data_to_update[rel_issue["number"]]["eventsList"].append(referenced_issue_event) - else: - ref = dict() - ref["number"] = rel_issue["number"] - ref["eventsList"] = list() - ref["eventsList"].append(referenced_issue_event) - issue_data_to_update[rel_issue["number"]] = ref - - # the format of every related commit is adjusted to the event format - for rel_commit in issue["relatedCommits"]: - - rel_commit["created_at"] = format_time(rel_commit["referenced_at"]) - rel_commit["event_info_1"] = rel_commit["commit"]["hash"] - rel_commit["event_info_2"] = "" - rel_commit["ref_target"] = "" - - # if the related commit is not of type "commit" but "commitAddedToPullRequest", - # it is a commit which was added to the pull request - if rel_commit["type"] == "commitAddedToPullRequest": - rel_commit["event"] = "commit_added" - - # if the related commit was mentioned in an issue comment: - elif rel_commit["type"] == "commitMentionedInIssue": - rel_commit["event"] = "add_link" - rel_commit["event_info_2"] = "commit" - - # else it is a commit which references issue/pull-request - else: - rel_commit["event"] = "referenced_by" - rel_commit["event_info_2"] = "commit" - - # the format of every comment is adjusted to the event format - for comment in issue["commentsList"]: - comment["event"] = "commented" - comment["ref_target"] = "" - comment["created_at"] = format_time(comment["referenced_at"]) - if "event_info_1" not in comment: - comment["event_info_1"] = "" - if "event_info_2" not in comment: - comment["event_info_2"] = "" - - # cache comment by date to resolve/re-arrange references later - comments[comment["created_at"]] = comment - - # the format of every review and their comments is adjusted to the event format - for review in issue["reviewsList"]: - review["event"] = "reviewed" - review["created_at"] = format_time(review["submitted_at"]) - review["event_info_1"] = review["state"].lower() - review["event_info_2"] = "" - review["ref_target"] = "" - - if review["hasReviewInitialComment"]: - initialComment = dict() - initialComment["event"] = "commented" - initialComment["user"] = review["user"] - initialComment["ref_target"] = "" - initialComment["created_at"] = format_time(review["submitted_at"]) - initialComment["event_info_1"] = "" - initialComment["event_info_2"] = "" - - issue["commentsList"].append(initialComment) - - # cache comment by date to resolve/re-arrange references later - comments[initialComment["created_at"]] = initialComment - - for reviewComment in review["reviewComments"]: - reviewComment["event"] = "commented" - reviewComment["created_at"] = format_time(reviewComment["referenced_at"]) - reviewComment["ref_target"] = "" - reviewComment["event_info_1"] = "" - reviewComment["event_info_2"] = "" - - # cache comment by date to resolve/re-arrange references later - comments[reviewComment["created_at"]] = reviewComment - - issue["commentsList"].append(reviewComment) - - # add dismissal comments to the list of comments - for event in issue["eventsList"]: - - if (event["event"] == "review_dismissed" and not event["dismissalMessage"] is None - and not event["dismissalMessage"] == ""): - dismissalComment = dict() - dismissalComment["event"] = "commented" - dismissalComment["user"] = event["user"] - dismissalComment["created_at"] = format_time(event["created_at"]) - dismissalComment["ref_target"] = "" - dismissalComment["event_info_1"] = "" - dismissalComment["event_info_2"] = "" + issue["discussion_begin"] = format_time(issue["discussion_begin"]) - # cache comment by date to resolve/re-arrange references later - comments[dismissalComment["created_at"]] = dismissalComment - - issue["commentsList"].append(dismissalComment) + # parses the close time in the correct format + issue["discussion_end"] = format_time(issue["discussion_end"]) - # the format of every event is adjusted - for event in issue["eventsList"]: - event["ref_target"] = "" - event["created_at"] = format_time(event["created_at"]) - if "event_info_1" not in event: - event["event_info_1"] = "" - if "event_info_2" not in event: - event["event_info_2"] = "" - - # if event collides with a comment - if event["created_at"] in comments: - comment = comments[event["created_at"]] - # if someone gets mentioned or subscribed by someone else in a comment, - # re-write the reference - if (event["event"] == "mentioned" or event["event"] == "subscribed") and \ - comment["event"] == "commented": - event["ref_target"] = event["user"] - event["user"] = comment["user"] - elif subtract_seconds_from_time(event["created_at"], 1) in comments: - comment = comments[subtract_seconds_from_time(event["created_at"], 1)] - # if someone gets mentioned or subscribed by someone else in a comment, - # re-write the reference - if (event["event"] == "mentioned" or event["event"] == "subscribed") and \ - comment["event"] == "commented": - event["ref_target"] = event["user"] - event["user"] = comment["user"] - - # if event is a referenced commit, we can update the user information - if event["event"] == "referenced" and event["commit"] is not None: - if (event["user"] is None): - event["user"] = dict() - event["user"]["name"] = event["commit"]["author"]["name"] # author or committer? - event["user"]["email"] = event["commit"]["author"]["email"] - event["user"]["username"] = event["commit"]["author"]["username"] - - # if event is a review request, we can update the ref target with the requested reviewer - if event["event"] == "review_requested" or event["event"] == "review_request_removed": - event["ref_target"] = event["requestedReviewer"] - - # if event dismisses a review, we can determine the original state of the corresponding review - if event["event"] == "review_dismissed": - for review in issue["reviewsList"]: - if review["reviewId"] == event["reviewId"]: - review["state"] = review["event_info_1"] = event["state"] - review["event_info_2"] = "later_dismissed" - - # if event is assign event, we have set the user to the assigner and the ref target to the assignee - if event["event"] == "assigned" or event["event"] == "unassigned": - event["ref_target"] = event["user"] - event["user"] = event["assigner"] - - # merge events, relatedCommits, relatedIssues and comment lists - issue["eventsList"] = issue["commentsList"] + issue["eventsList"] + issue["relatedIssues"] + issue[ - "relatedCommits"] + issue["reviewsList"] - - # remove events without user - issue["eventsList"] = [event for event in issue["eventsList"] if - not (event["user"] is None or event["ref_target"] is None)] - - # sorts eventsList by time - issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) - - # updates all the issues by the temporarily stored referenced_by events - for key, value in issue_data_to_update.iteritems(): - for issue in issue_data: - if issue["number"] == value["number"]: - issue["eventsList"] = issue["eventsList"] + value["eventsList"] + # checks if the issue is a pull-request or a normal issue and adapts the type + issue["type"].append("issue") return issue_data -def reformat_events(issue_data): - """ - Re-format event information dependent on the event type. - - :param issue_data: the data of all issues that shall be re-formatted - :return: the issue data with updated event information - """ - - log.info("Update event information ...") - - users = dict() - - # create a dictionary of users to merge GitHub usernames and names and e-mails originating from the git repository - for issue in issue_data: - - for event in issue["eventsList"]: - - # 1) add or update users which are authors of commits - # (committers of commits are usually the actor of the current event and will be dealt with in part 2 below) - if (event["event"] == "commit_added" or (event["event"] == "add_link" and event["event_info_2"] == "commit") - or (event["event"] == "referenced_by" and event["event_info_2"] == "commit")): - users = update_user_dict(users, event["commit"]["author"]) - - # 2) add or update users which are actor of the current event - users = update_user_dict(users, event["user"]) - - # 3) add or update users which are ref_target of the current event - if not event["ref_target"] is None and not event["ref_target"] == "": - users = update_user_dict(users, event["ref_target"]) - - # as the user dictionary is created, start re-formating the event information of all issues - for issue in issue_data: - - events_to_remove = list() - - # re-format information of every event in the eventsList of an issue - for event in issue["eventsList"]: - - # lookup user in dictionary - event["user"] = lookup_user(users, event["user"]) - if (event["ref_target"] != ""): - event["ref_target"] = lookup_user(users, event["ref_target"]) - - - if event["event"] == "closed": - event["event"] = "state_updated" - event["event_info_1"] = "closed" # new state - event["event_info_2"] = "open" # old state - issue["state_new"] = "closed" - - elif event["event"] == "reopened": - event["event"] = "state_updated" - event["event_info_1"] = "open" # new state - event["event_info_2"] = "closed" # old state - issue["state_new"] = "reopened" - - elif event["event"] == "labeled": - label = event["label"]["name"].lower() - event["event_info_1"] = label - - # if the label is in this list, it also is a type of the issue - if label in known_types: - issue["type"].append(str(label)) - - # creates an event for type updates and adds it to the eventsList - type_event = dict() - type_event["user"] = event["user"] - type_event["created_at"] = event["created_at"] - type_event["event"] = "type_updated" - type_event["event_info_1"] = label - type_event["event_info_2"] = "" - type_event["ref_target"] = "" - issue["eventsList"].append(type_event) - - # if the label is in this list, it also is a resolution of the issue - elif label in known_resolutions: - issue["resolution"].append(str(label)) - - # creates an event for resolution updates and adds it to the eventsList - resolution_event = dict() - resolution_event["user"] = event["user"] - resolution_event["created_at"] = event["created_at"] - resolution_event["event"] = "resolution_updated" - resolution_event["event_info_1"] = label - resolution_event["event_info_2"] = "" - resolution_event["ref_target"] = "" - issue["eventsList"].append(resolution_event) - - elif event["event"] == "unlabeled": - label = event["label"]["name"].lower() - event["event_info_1"] = label - - # if the label is in this list, it also is a type of the issue - if label in known_types: - if label in issue["type"]: - issue["type"].remove(str(label)) - - # creates an event for type updates and adds it to the eventsList - type_event = dict() - type_event["user"] = event["user"] - type_event["created_at"] = event["created_at"] - type_event["event"] = "type_updated" - type_event["event_info_1"] = "" - type_event["event_info_2"] = label - type_event["ref_target"] = "" - issue["eventsList"].append(type_event) - - # if the label is in this list, it also is a resolution of the issue - elif label in known_resolutions: - if label in issue["resolution"]: - issue["resolution"].remove(str(label)) - - # creates an event for resolution updates and adds it to the eventsList - resolution_event = dict() - resolution_event["user"] = event["user"] - resolution_event["created_at"] = event["created_at"] - resolution_event["event"] = "resolution_updated" - resolution_event["event_info_1"] = "" - resolution_event["event_info_2"] = label - resolution_event["ref_target"] = "" - issue["eventsList"].append(resolution_event) - - elif event["event"] == "commented": - # "state_new" and "resolution" of the issue give the information about the state and the resolution of - # the issue when the comment was written, because the eventsList is sorted by time - event["event_info_1"] = issue["state_new"] - event["event_info_2"] = issue["resolution"] - - elif event["event"] == "referenced" and not event["commit"] is None: - # remove "referenced" events originating from commits - # as they are handled as referenced commit - events_to_remove.append(event) - - # sorts eventsList by time again - issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) - - # remove unwanted events - for event_to_remove in events_to_remove: - issue["eventsList"].remove(event_to_remove) - - return issue_data def insert_user_data(issues, conf, resdir): @@ -743,7 +448,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): for username in username_id_buffer: user = get_user_from_id(username_id_buffer[username]) lines.append(( - username, + # username, user["name"], user["email"] )) From a73461a7e59d3c8f9b9b604f26f0c771b434c822 Mon Sep 17 00:00:00 2001 From: RitikaHiremath <67013490+RitikaHiremath@users.noreply.github.com> Date: Mon, 8 Dec 2025 12:14:22 +0100 Subject: [PATCH 39/79] Delete .DS_Store --- .DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store deleted file mode 100644 index 31e9fb2ab626563d82e455aa0db2cbe3420f0d1a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHKQA^xF5S~5HnnQ$u6?_czRq)hek-mh}{((m7Ln~==i8eIN9!aiOI1cjM{-OSo z{vN;C-4o?nsC_6RGso;VnVre*eiL?YiAas6*#VJHL>`o}H$d|n;eOT?={e6jP?>jR zlu!$eDPM^e$75uGYqyPxO{hTC?$+-cDp{jv3@k`fl{Mos)w<+mY{RV)p3{{H$xX>u zQqoy7NlTp~<7YZ4i>4h8f2rPDfBo5px8ZGim(eFZi`t}}Hlw6|&rjzVB3;@^`mV@l zvH$#3S8Y;Md2JGkEJw(Pi=xW(Y^0}{o5_uAhd1yBvA;W?zd9N21;1&vT`>RWk$#Q z0~1avv?#4GAPn4RV9h@IeEz@r^ZoyRk~|3m!oZVaK=sd}vk{i$&eoO1@mU){KSEhJ nuCsWX0>d1|h~=Yr7it9lfDK^iu(JpcME(dE8l({h{wo7FyMA?* From 43764431a6afe0ced587f29bf268c81760123994 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Sun, 21 Dec 2025 01:26:09 +0100 Subject: [PATCH 40/79] python3 updated in Zulip issue processing --- .../old_zulip_issue_processing.py | 488 ++++++++++++++++++ issue_processing/zulip_issue_processing.py | 153 +++--- 2 files changed, 557 insertions(+), 84 deletions(-) create mode 100644 issue_processing/old_zulip_issue_processing.py diff --git a/issue_processing/old_zulip_issue_processing.py b/issue_processing/old_zulip_issue_processing.py new file mode 100644 index 0000000..9a57402 --- /dev/null +++ b/issue_processing/old_zulip_issue_processing.py @@ -0,0 +1,488 @@ +# new zulip issue processing + +""" +This file is able to extract Zulip issue data from json files. +""" + +import argparse +import httplib +import json +import os +import sys +import urllib +from datetime import datetime, timedelta +from logging import getLogger +import base64 + +import operator +from codeface_utils.cluster.idManager import dbIdManager, csvIdManager +from codeface_utils.configuration import Configuration +from codeface_utils.dbmanager import DBManager +from dateutil import parser as dateparser +from datetime import datetime + +from csv_writer import csv_writer + +# datetime format string +datetime_format = "%Y-%m-%d %H:%M:%S" + +def run(): + # get data from zulip api calls . then format it and apply to codeface extraction + # get all needed paths and arguments for the method call. + parser = argparse.ArgumentParser(prog='codeface-extraction-issues-zulip', description='Codeface extraction') + parser.add_argument('-c', '--config', help="Codeface configuration file", default='codeface.conf') + parser.add_argument('-p', '--project', help="Project configuration file", required=True) + parser.add_argument('resdir', help="Directory to store analysis results in") + + # parse arguments + args = parser.parse_args(sys.argv[1:]) + __codeface_conf, __project_conf = map(os.path.abspath, (args.config, args.project)) + + # create configuration + __conf = Configuration.load(__codeface_conf, __project_conf) + + # get source and results folders + __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) + __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) + + # run processing of issue data: + # 1) load the list of issues + issues = load(__srcdir) + # 2) update missing colums + issues = update(issues) + # 3) re-format the issues + issues = reformat_issues(issues) + # 5) update user data with Codeface database and dump username-to-name/e-mail list + issues = insert_user_data(issues, __conf, __resdir) + # 6) dump result to disk + print_to_disk(issues, __resdir) + + log.info("Zulip issue processing complete!") + + + +def load(source_folder): + """Load issues from disk. + + :param source_folder: the folder where to find 'issues.json' + :return: the loaded issue data + """ + + srcfile = os.path.join(source_folder, "issues.json") + log.info("Loading Github issues from file '{}'...".format(srcfile)) + + # check if file exists and exit early if not + if not os.path.exists(srcfile): + log.error("Zulip issue file '{}' does not exist! Exiting early...".format(srcfile)) + sys.exit(-1) + + with open(srcfile) as issues_file: + issue_data = json.load(issues_file) + + return issue_data + +#UPDATED +def format_time(time): + """ + Format times from different sources to a consistent time format + + :param time: the time that shall be formatted + :return: the formatted time + """ + + # empty time would be formatted to current date + if time == "" or time is None: + return "" + else: + d = datetime.fromtimestamp(time) + return d.strftime(datetime_format) + + +def subtract_seconds_from_time(time, seconds): + """ + Subtract the specified number of seconds from a date string + + :param time: the date string to subtract the specified seconds from + :param seconds: the number of seconds to subtract from the date string + :return: the date string after subtracting the specified number of seconds + """ + + new_time = datetime.strptime(time, datetime_format) - timedelta(seconds = seconds) + return new_time.strftime(datetime_format) + + +def create_user(name, username, email): + """ + Creates a user object with all needed information + + :param name: the name the user shall have + :param username: the username the user shall have + :param email: the email the user shall have + :return: the created user object + """ + + if name is None: + name = "" + if username is None: + username = "" + if email is None: + email = "" + + user = dict() + user["name"] = name + user["username"] = username + user["email"] = email + + return user + + +def create_deleted_user(): + """ + Creates a user object for a deleted user (ghost user) + + :return: the created user object for a deleted user + """ + + return create_user("Deleted user", "ghost", "ghost@github.com") + + +def lookup_user(user_dict, user): + """ + Alters a user object in the case that name or email are missing by the corresponding name and email + from a given user dictionary + + :param user_dict: the user dictionary + :param user: the user object to lookup in the dictionary + :return: the altered user object in case of a lookup + or the unaltered user object otherwise + """ + + # if user is None, replace it by a deleted user + if user is None: + user = create_deleted_user() + + if (user["name"] == "" or user["name"] is None or + user["email"] is None or user["email"] == ""): + + # lookup user only if username is not None and not empty + if not user["username"] is None and not user["username"] == "": + user = user_dict[user["username"]] + + return user + + +def update_user_dict(user_dict, user): + """ + Adds or updates users to merge GitHub usernames and names and e-mail addresses originating from the git repository + + :param user_dict: the user dictionary + :param user: the user object to add to or update in the dictionary + :return: the updated user dictionary + """ + + # if the given user is None, use the deleted user instead + if user is None: + user = create_deleted_user() + + if not user["username"] in user_dict.keys(): + if not user["username"] is None and not user["username"] == "": + user_dict[user["username"]] = user + else: + user_in_dict = user_dict[user["username"]] + if user_in_dict["name"] is None or user_in_dict["name"] == "": + user_in_dict["name"] = user["name"] + if user_in_dict["email"] is None or user_in_dict["email"] == "": + user_in_dict["email"] = user["email"] + user_dict[user["username"]] = user_in_dict + + return user_dict + +def discussion_id_update(issue_data): + """ + :param issue_data: + """ + # Group timestamps by discussion_topic + grouped = {} + + for item in issue_data: + topic = item["discusssion_topic"] + if topic not in grouped: + grouped[topic] = [] + grouped[topic].append(item) + + # Process each topic group + for topic, messages in grouped.items(): + + # 1. Sort messages by timestamp ASC + messages.sort(key=lambda m: m["timestamp"]) + + # 2. Add sequential discussion_id suffix (#1, #2, #3...) + for idx, msg in enumerate(messages, start=1): + msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' + + return issue_data + +def discussion_begin_end_add(issue_data): + """ + :param issue_data: the issue data where discussion_begin and discussion_end time is to be added + :return: the updated data + """ + # Group timestamps by discussion_topic + discussion_topics = {} + + for item in issue_data: + d_topic = item["discussion_topic"] + ts = item["timestamp"] + + if d_topic not in discussion_topics: + discussion_topics[d_topic] = [] + discussion_topics[d_topic].append(ts) + + # Compute begin and end for each discussion + discussion_bounds = { + d_topic: { + "discussion_begin": min(times), + "discussion_end": max(times) + } + for d_topic, times in discussion_topics.items() + } + + # Add begin/end back to each entry + for item in issue_data: + d_topic = item["discussion_topic"] + item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] + item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] + + return issue_data + +def update(issue_data): + """ + updates values in the issue data structure as per requirement. + :params: issue_data: the issue data to be updated.x + :return: returns the issue data. + """ + # updating id of issues in a particular format. + issue_data = discussion_id_update(issue_data) + # adding discussion begin and end time for each issue data. + issue_data = discussion_begin_end_add(issue_data) + + + return issue_data + +def reformat_issues(issue_data): + """ + Re-arrange issue data structure. + + :param issue_data: the issue data to re-arrange + :return: the re-arranged issue data + """ + + log.info("Re-arranging Github issues...") + + # re-process all issues + for issue in issue_data: + + # empty container for issue types + issue["type"] = [] + + # empty container for issue resolutions + issue["resolution"] = [] + + + # if an issue has no relatedCommits, an empty List gets created + if issue["relatedCommits"] is None: + issue["relatedCommits"] = [] + + # if an issue has no relatedIssues, an empty List gets created + if "relatedIssues" not in issue: + issue["relatedIssues"] = [] + + # add "closed_at" information if not present yet + # if issue["closed_at"] is None: + # issue["closed_at"] = "" + + # parses the creation time in the correct format + # issue["created_at"] = format_time(issue["created_at"]) + + # parses the close time in the correct format + # issue["closed_at"] = format_time(issue["closed_at"]) + + issue["discussion_begin"] = format_time(issue["discussion_begin"]) + + # parses the close time in the correct format + issue["discussion_end"] = format_time(issue["discussion_end"]) + + # checks if the issue is a pull-request or a normal issue and adapts the type + issue["type"].append("issue") + + return issue_data + + + + +def insert_user_data(issues, conf, resdir): + """ + Insert user data into database and update issue data. + In addition, dump username-to-user list to file. + + :param issues: the issues to retrieve user data from + :param conf: the project configuration + :param resdir: the directory in which the username-to-user-list should be dumped + :return: the updated issue data + """ + + log.info("Syncing users with ID service...") + + # create buffer for users (key: user id) + user_buffer = dict() + # create buffer for user ids (key: user string) + user_id_buffer = dict() + # create buffer for usernames (key: username) + username_id_buffer = dict() + # open database connection + dbm = DBManager(conf) + # open ID-service connection + idservice = idManager(dbm, conf) + + def get_user_string(name, email): + if not email or email is None: + return "{name}".format(name=name) + # return "{name} <{name}@default.com>".format(name=name) # for debugging only + else: + return "{name} <{email}>".format(name=name, email=email) + + def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames=username_id_buffer): + username = unicode(user["username"]).encode("utf-8") + + # fix encoding for name and e-mail address + if user["name"] is not None: + name = unicode(user["name"]).encode("utf-8") + else: + name = username + mail = unicode(user["email"]).encode("utf-8") + # construct string for ID service and send query + user_string = get_user_string(name, mail) + + # check buffer to reduce amount of DB queries + if user_string in buffer_db_ids: + log.devinfo("Returning person id for user '{}' from buffer.".format(user_string)) + if username is not None: + buffer_usernames[username] = buffer_db_ids[user_string] + return buffer_db_ids[user_string] + + # get person information from ID service + log.devinfo("Passing user '{}' to ID service.".format(user_string)) + idx = idservice.getPersonID(user_string) + + # add user information to buffer + # user_string = get_user_string(user["name"], user["email"]) # update for + buffer_db_ids[user_string] = idx + + # add id to username buffer + if username is not None: + buffer_usernames[username] = idx + + return idx + + def get_user_from_id(idx, buffer_db=user_buffer): + + # check whether user information is in buffer to reduce amount of DB queries + if idx in buffer_db: + log.devinfo("Returning user '{}' from buffer.".format(idx)) + return buffer_db[idx] + + # get person information from ID service + log.devinfo("Passing user id '{}' to ID service.".format(idx)) + person = idservice.getPersonFromDB(idx) + user = dict() + user["email"] = person["email1"] # column "email1" + user["name"] = person["name"] # column "name" + user["id"] = person["id"] # column "id" + + # add user information to buffer + buffer_db[idx] = user + + return user + + + # check and update database for all occurring users + for issue in issues: + # check database for issue author + issue["user"] = get_id_and_update_user(issue["user"]) + + # check database for event authors + for event in issue["eventsList"]: + event["user"] = get_id_and_update_user(event["user"]) + + # check database for the reference-target user if needed + if event["ref_target"] != "": + event["ref_target"] = get_id_and_update_user(event["ref_target"]) + + # get all users after database updates having been performed + for issue in issues: + # get issue author + issue["user"] = get_user_from_id(issue["user"]) + + # get event authors + for event in issue["eventsList"]: + event["user"] = get_user_from_id(event["user"]) + + # get the reference-target user if needed + if event["ref_target"] != "": + event["ref_target"] = get_user_from_id(event["ref_target"]) + event["event_info_1"] = event["ref_target"]["name"] + event["event_info_2"] = event["ref_target"]["email"] + + # dump username, name, and e-mail to file + lines = [] + for username in username_id_buffer: + user = get_user_from_id(username_id_buffer[username]) + lines.append(( + # username, + user["name"], + user["email"] + )) + + log.info("Dump username list to file...") + username_dump = os.path.join(resdir, "usernames.list") + csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) + + return issues + + +def print_to_disk(issues, results_folder): + """ + Print issues to file "issues-github.list" in the results folder. + + :param issues: the issues to dump + :param results_folder: the folder where to place "issues-github.list" output file + """ + + # construct path to output file + output_file = os.path.join(results_folder, "issues-github.list") + log.info("Dumping output in file '{}'...".format(output_file)) + + # construct lines of output + lines = [] + for issue in issues: + for event in issue["eventsList"]: + lines.append(( + issue["number"], + issue["title"], + json.dumps(issue["type"]), + issue["state_new"], + json.dumps(issue["resolution"]), + issue["created_at"], + issue["closed_at"], + json.dumps([]), # components + event["event"], + event["user"]["name"], + event["user"]["email"], + event["created_at"], + event["event_info_1"], + json.dumps(event["event_info_2"]) + )) + + # write to output file + csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) + diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 645cef4..0878b99 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -1,43 +1,60 @@ -# new zulip issue processing - +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2017 by Raphael Nömmer +# Copyright 2017 by Claus Hunsen +# Copyright 2018 by Barbara Eckl +# Copyright 2018-2019 by Anselm Fehnker +# Copyright 2019 by Thomas Bock +# Copyright 2020-2021 by Thomas Bock +# Copyright 2025 by Maximilian Löffler +# Copyright 2025 by Ritika Hiremath +# All Rights Reserved. """ This file is able to extract Zulip issue data from json files. """ -# import argparse -# import httplib +import argparse import json import os import sys -# import urllib from datetime import datetime, timedelta -import hashlib -import base64 - -import operator -from codeface.cli import log -from codeface.cluster.idManager import idManager -from codeface.configuration import Configuration -from codeface.dbmanager import DBManager +from logging import getLogger + +from codeface_utils.cluster.idManager import dbIdManager, csvIdManager +from codeface_utils.configuration import Configuration +from codeface_utils.dbmanager import DBManager from dateutil import parser as dateparser -from datetime import datetime from csv_writer import csv_writer + +log = getLogger(__name__) + # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" def run(): - # get data from zulip api calls . then format it and apply to codeface extraction # get all needed paths and arguments for the method call. - parser = argparse.ArgumentParser(prog='codeface-extraction-issues-zulip', description='Codeface extraction') + parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') parser.add_argument('-c', '--config', help="Codeface configuration file", default='codeface.conf') parser.add_argument('-p', '--project', help="Project configuration file", required=True) parser.add_argument('resdir', help="Directory to store analysis results in") # parse arguments args = parser.parse_args(sys.argv[1:]) - __codeface_conf, __project_conf = map(os.path.abspath, (args.config, args.project)) + __codeface_conf, __project_conf = list(map(os.path.abspath, (args.config, args.project))) # create configuration __conf = Configuration.load(__codeface_conf, __project_conf) @@ -49,6 +66,7 @@ def run(): # run processing of issue data: # 1) load the list of issues issues = load(__srcdir) + # 2) re-format the issues # 2) update missing colums issues = update(issues) # 3) re-format the issues @@ -61,7 +79,6 @@ def run(): log.info("Zulip issue processing complete!") - def load(source_folder): """Load issues from disk. @@ -70,11 +87,11 @@ def load(source_folder): """ srcfile = os.path.join(source_folder, "issues.json") - log.devinfo("Loading Github issues from file '{}'...".format(srcfile)) + log.info("Loading Github issues from file '{}'...".format(srcfile)) # check if file exists and exit early if not if not os.path.exists(srcfile): - log.error("Zulip issue file '{}' does not exist! Exiting early...".format(srcfile)) + log.error("Github issue file '{}' does not exist! Exiting early...".format(srcfile)) sys.exit(-1) with open(srcfile) as issues_file: @@ -82,7 +99,7 @@ def load(source_folder): return issue_data -#UPDATED + def format_time(time): """ Format times from different sources to a consistent time format @@ -166,12 +183,11 @@ def lookup_user(user_dict, user): user["email"] is None or user["email"] == ""): # lookup user only if username is not None and not empty - if not user["username"] is None and not user["username"] == "": + if user["username"] is not None and not user["username"] == "": user = user_dict[user["username"]] return user - def update_user_dict(user_dict, user): """ Adds or updates users to merge GitHub usernames and names and e-mail addresses originating from the git repository @@ -185,8 +201,8 @@ def update_user_dict(user_dict, user): if user is None: user = create_deleted_user() - if not user["username"] in user_dict.keys(): - if not user["username"] is None and not user["username"] == "": + if user["username"] not in list(user_dict.keys()): + if user["username"] is not None and not user["username"] == "": user_dict[user["username"]] = user else: user_in_dict = user_dict[user["username"]] @@ -202,7 +218,6 @@ def discussion_id_update(issue_data): """ :param issue_data: """ - # Group timestamps by discussion_topic grouped = {} for item in issue_data: @@ -211,13 +226,9 @@ def discussion_id_update(issue_data): grouped[topic] = [] grouped[topic].append(item) - # Process each topic group for topic, messages in grouped.items(): - # 1. Sort messages by timestamp ASC messages.sort(key=lambda m: m["timestamp"]) - - # 2. Add sequential discussion_id suffix (#1, #2, #3...) for idx, msg in enumerate(messages, start=1): msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' @@ -228,7 +239,7 @@ def discussion_begin_end_add(issue_data): :param issue_data: the issue data where discussion_begin and discussion_end time is to be added :return: the updated data """ - # Group timestamps by discussion_topic + discussion_topics = {} for item in issue_data: @@ -239,7 +250,6 @@ def discussion_begin_end_add(issue_data): discussion_topics[d_topic] = [] discussion_topics[d_topic].append(ts) - # Compute begin and end for each discussion discussion_bounds = { d_topic: { "discussion_begin": min(times), @@ -248,7 +258,6 @@ def discussion_begin_end_add(issue_data): for d_topic, times in discussion_topics.items() } - # Add begin/end back to each entry for item in issue_data: d_topic = item["discussion_topic"] item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] @@ -267,6 +276,8 @@ def update(issue_data): return issue_data + + def reformat_issues(issue_data): """ Re-arrange issue data structure. @@ -275,7 +286,7 @@ def reformat_issues(issue_data): :return: the re-arranged issue data """ - log.devinfo("Re-arranging Github issues...") + log.info("Re-arranging Github issues...") # re-process all issues for issue in issue_data: @@ -286,50 +297,23 @@ def reformat_issues(issue_data): # empty container for issue resolutions issue["resolution"] = [] - # TO DO: Are these column names needed? though they will be empty - # if an issue has no eventsList, an empty List gets created - if issue["eventsList"] is None: - issue["eventsList"] = [] - - # if an issue has no commentsList, an empty List gets created - if issue["commentsList"] is None: - issue["commentsList"] = [] - # if an issue has no relatedCommits, an empty List gets created if issue["relatedCommits"] is None: issue["relatedCommits"] = [] - # if an issue has no reviewsList, an empty Listgets created - if issue["reviewsList"] is None: - issue["reviewsList"] = [] - # if an issue has no relatedIssues, an empty List gets created if "relatedIssues" not in issue: issue["relatedIssues"] = [] - # add "closed_at" information if not present yet - # if issue["closed_at"] is None: - # issue["closed_at"] = "" - - # parses the creation time in the correct format - # issue["created_at"] = format_time(issue["created_at"]) - - # parses the close time in the correct format - # issue["closed_at"] = format_time(issue["closed_at"]) - issue["discussion_begin"] = format_time(issue["discussion_begin"]) # parses the close time in the correct format issue["discussion_end"] = format_time(issue["discussion_end"]) - # checks if the issue is a pull-request or a normal issue and adapts the type issue["type"].append("issue") return issue_data - - - def insert_user_data(issues, conf, resdir): """ Insert user data into database and update issue data. @@ -349,10 +333,13 @@ def insert_user_data(issues, conf, resdir): user_id_buffer = dict() # create buffer for usernames (key: username) username_id_buffer = dict() - # open database connection - dbm = DBManager(conf) - # open ID-service connection - idservice = idManager(dbm, conf) + + # connect to ID service + if conf["useCsv"]: + idservice = csvIdManager(conf) + else: + dbm = DBManager(conf) + idservice = dbIdManager(dbm, conf) def get_user_string(name, email): if not email or email is None: @@ -362,26 +349,24 @@ def get_user_string(name, email): return "{name} <{email}>".format(name=name, email=email) def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames=username_id_buffer): - username = unicode(user["username"]).encode("utf-8") - # fix encoding for name and e-mail address - if user["name"] is not None: - name = unicode(user["name"]).encode("utf-8") - else: - name = username - mail = unicode(user["email"]).encode("utf-8") + # ensure string representation for name and e-mail address + username = str(user["username"]) + name = str(user["name"]) if "name" in user else username + mail = str(user["email"]) + # construct string for ID service and send query user_string = get_user_string(name, mail) # check buffer to reduce amount of DB queries if user_string in buffer_db_ids: - log.devinfo("Returning person id for user '{}' from buffer.".format(user_string)) + log.info("Returning person id for user '{}' from buffer.".format(user_string)) if username is not None: buffer_usernames[username] = buffer_db_ids[user_string] return buffer_db_ids[user_string] # get person information from ID service - log.devinfo("Passing user '{}' to ID service.".format(user_string)) + log.info("Passing user '{}' to ID service.".format(user_string)) idx = idservice.getPersonID(user_string) # add user information to buffer @@ -398,16 +383,17 @@ def get_user_from_id(idx, buffer_db=user_buffer): # check whether user information is in buffer to reduce amount of DB queries if idx in buffer_db: - log.devinfo("Returning user '{}' from buffer.".format(idx)) + log.info("Returning user '{}' from buffer.".format(idx)) return buffer_db[idx] # get person information from ID service - log.devinfo("Passing user id '{}' to ID service.".format(idx)) + log.info("Passing user id '{}' to ID service.".format(idx)) person = idservice.getPersonFromDB(idx) - user = dict() - user["email"] = person["email1"] # column "email1" - user["name"] = person["name"] # column "name" - user["id"] = person["id"] # column "id" + user = { + "name": person["name"], + "email": person["email1"], + "id": person["id"] + } # add user information to buffer buffer_db[idx] = user @@ -448,7 +434,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): for username in username_id_buffer: user = get_user_from_id(username_id_buffer[username]) lines.append(( - # username, + username, user["name"], user["email"] )) @@ -459,7 +445,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): return issues - +# TO DO def print_to_disk(issues, results_folder): """ Print issues to file "issues-github.list" in the results folder. @@ -494,5 +480,4 @@ def print_to_disk(issues, results_folder): )) # write to output file - csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) - + csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) \ No newline at end of file From bff13d2da13dfffc8367d5a12ce4d294d31558b1 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 22 Dec 2025 13:47:09 +0100 Subject: [PATCH 41/79] added bot notifications functions --- issue_processing/zulip_issue_processing.py | 62 +++++++++++++++++++++- 1 file changed, 60 insertions(+), 2 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 0878b99..ff0552a 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -265,6 +265,65 @@ def discussion_begin_end_add(issue_data): return issue_data +def bot_event_type(issue): + """ + Docstring for bot_event_type + + :param issue: Description + """ + content = issue.get("content", "").lower() + + if "stream created" in content: + return "stream created" + + if "changed the description" in content: + return "stream description changed" + + if "changed the access permissions" in content: + return "stream permissions changed" + + + if "wave" in content: + return "wave" + + return "unclassified event" + +def notification_bot_event(issue): + """ + Docstring for notification_bot_event + + :param issue: Description + """ + content = issue.get("content", "").lower() + if "has marked this topic as resolved" in content: + return "topic resolved" + + if "has marked this topic as unresolved" in content: + return "topic unresolved" + + if "topic was moved" in content: + return "topic moved" + + +def event_type(issue_data): + """ + Docstring for event_type + + :param issue_data: Description + """ + + for issue in issue_data: + if(("stream events" in issue["discussion_topic"])): + # update_user here + issue["individual_events"]= bot_event_type(issue) + else: + if(issue["sender_full_name"] == "Notification Bot"): + issue["individual_events"] = notification_bot_event(issue) + + issue["individual_events"]= "commented event" + + return issue_data + def update(issue_data): """ updates values in the issue data structure as per requirement. @@ -273,11 +332,10 @@ def update(issue_data): """ issue_data = discussion_id_update(issue_data) issue_data = discussion_begin_end_add(issue_data) + issue_data = event_type(issue_data) return issue_data - - def reformat_issues(issue_data): """ Re-arrange issue data structure. From 903acf2910f083b3841609008acd890f0b600c6d Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 22 Dec 2025 15:32:10 +0100 Subject: [PATCH 42/79] removed extra files from issue_processing/ --- issue_processing/trial.py | 31 ------------------------------- 1 file changed, 31 deletions(-) delete mode 100644 issue_processing/trial.py diff --git a/issue_processing/trial.py b/issue_processing/trial.py deleted file mode 100644 index f4ad744..0000000 --- a/issue_processing/trial.py +++ /dev/null @@ -1,31 +0,0 @@ -import json - -def discussion_id_update(issue_data): - """ - """ - # Group timestamps by discussion_topic - grouped = {} - - for item in issue_data: - topic = item["discussion_topic"] - if topic not in grouped: - grouped[topic] = [] - grouped[topic].append(item) - - # Process each topic group - for topic, messages in grouped.items(): - - # 1. Sort messages by timestamp ASC - messages.sort(key=lambda m: m["timestamp"]) - - # 2. Add sequential discussion_id suffix (#1, #2, #3...) - for idx, msg in enumerate(messages, start=1): - msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' - - return issue_data - - -with open("/Users/ritikahiremath/Desktop/extraction/issue_processing/_issues/_issues.json", "r") as f: - issue_data = json.load(f) -discussion_id_update(issue_data) -print(issue_data) \ No newline at end of file From 78eb0a98ae1cfd4565468ecc2fbe980787516fad Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 11:29:29 +0100 Subject: [PATCH 43/79] bot based usernamefunction updated # Conflicts: # issue_processing/zulip_issue_processing.py --- issue_processing/zulip_issue_processing.py | 62 ++++++++++++++-------- 1 file changed, 40 insertions(+), 22 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index ff0552a..7ecebdd 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -36,6 +36,7 @@ from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager from dateutil import parser as dateparser +from bs4 import BeautifulSoup from csv_writer import csv_writer @@ -72,7 +73,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) + # issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) @@ -281,8 +282,10 @@ def bot_event_type(issue): if "changed the access permissions" in content: return "stream permissions changed" - + if "renamed stream" in content: + return "stream renamed" + if "wave" in content: return "wave" @@ -304,6 +307,20 @@ def notification_bot_event(issue): if "topic was moved" in content: return "topic moved" + return "user event" +def bot_event_name_update(issue): + """ + Docstring for bot_event_name_update + + :param issue: Description + """ + soup = BeautifulSoup(issue["content"], "html.parser") + mention = soup.find("span", class_="user-mention") + + if mention: + return mention.get_text(strip=True) + + return None def event_type(issue_data): """ @@ -313,11 +330,12 @@ def event_type(issue_data): """ for issue in issue_data: - if(("stream events" in issue["discussion_topic"])): - # update_user here + if(("stream events" in issue["discusssion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): issue["individual_events"]= bot_event_type(issue) + issue["sender_full_name"] = bot_event_name_update(issue) + issue["sender_email"] = None else: - if(issue["sender_full_name"] == "Notification Bot"): + if("stream events" in issue["discusssion_topic"]): issue["individual_events"] = notification_bot_event(issue) issue["individual_events"]= "commented event" @@ -368,6 +386,8 @@ def reformat_issues(issue_data): # parses the close time in the correct format issue["discussion_end"] = format_time(issue["discussion_end"]) + issue["timestamp"] = format_time(issue["timestamp"]) + issue["type"].append("issue") return issue_data @@ -503,7 +523,6 @@ def get_user_from_id(idx, buffer_db=user_buffer): return issues -# TO DO def print_to_disk(issues, results_folder): """ Print issues to file "issues-github.list" in the results folder. @@ -519,22 +538,21 @@ def print_to_disk(issues, results_folder): # construct lines of output lines = [] for issue in issues: - for event in issue["eventsList"]: - lines.append(( - issue["number"], - issue["title"], - json.dumps(issue["type"]), - issue["state_new"], - json.dumps(issue["resolution"]), - issue["created_at"], - issue["closed_at"], - json.dumps([]), # components - event["event"], - event["user"]["name"], - event["user"]["email"], - event["created_at"], - event["event_info_1"], - json.dumps(event["event_info_2"]) + lines.append(( + issue["discussion_id"], + issue["discusssion_topic"], + json.dumps(issue["type"]), + json.dumps([]), + json.dumps(issue["resolution"]), + issue["discussion_begin"], + issue["discussion_end"], + json.dumps([]), # components + issue["individual_events"], + issue["sender_full_name"], + issue["sender_email"], + issue["timestamp"], + json.dumps([]), + json.dumps([]) )) # write to output file From f6fe9fd2441fe73add5319f28612d8a42035b57b Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 19 Jan 2026 11:19:41 +0100 Subject: [PATCH 44/79] added comments to zulip-issue processing --- issue_processing/zulip_issue_processing.py | 74 ++++++++++++++-------- 1 file changed, 47 insertions(+), 27 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 7ecebdd..cad61af 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -73,7 +73,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - # issues = insert_user_data(issues, __conf, __resdir) + issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) @@ -83,16 +83,16 @@ def run(): def load(source_folder): """Load issues from disk. - :param source_folder: the folder where to find 'issues.json' - :return: the loaded issue data + :param source_folder: the folder where to find 'zulip.json' + :return: the loaded zulip data """ - srcfile = os.path.join(source_folder, "issues.json") - log.info("Loading Github issues from file '{}'...".format(srcfile)) + srcfile = os.path.join(source_folder, "zulip.json") + log.info("Loading Zulip data from file '{}'...".format(srcfile)) # check if file exists and exit early if not if not os.path.exists(srcfile): - log.error("Github issue file '{}' does not exist! Exiting early...".format(srcfile)) + log.error("Zulip data file '{}' does not exist! Exiting early...".format(srcfile)) sys.exit(-1) with open(srcfile) as issues_file: @@ -217,18 +217,22 @@ def update_user_dict(user_dict, user): def discussion_id_update(issue_data): """ - :param issue_data: + Updates the id for each issue data in zulip. + The ID is dependent on the topic id and followed by # and + the count of the message from the beginning. + :param issue_data: total issue data from zulip + : return: The updated issue data with the id updated. """ grouped = {} - + # groupds each discussion topic together to update the id for item in issue_data: topic = item["discusssion_topic"] if topic not in grouped: grouped[topic] = [] grouped[topic].append(item) + # Updates the disucssion id here. for topic, messages in grouped.items(): - messages.sort(key=lambda m: m["timestamp"]) for idx, msg in enumerate(messages, start=1): msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' @@ -237,12 +241,13 @@ def discussion_id_update(issue_data): def discussion_begin_end_add(issue_data): """ - :param issue_data: the issue data where discussion_begin and discussion_end time is to be added - :return: the updated data + Updates the discussion begin and end time for each discussion topic. + :param issue_data: Total issue data + :return: the updated issue data with two new columns. """ discussion_topics = {} - + # groups discussion topics and finds the max and min time that topic was discussed. for item in issue_data: d_topic = item["discussion_topic"] ts = item["timestamp"] @@ -268,12 +273,15 @@ def discussion_begin_end_add(issue_data): def bot_event_type(issue): """ - Docstring for bot_event_type + Updates the type of event . + This function only updates events notification bot username "Notification Bot". - :param issue: Description + :param issue: the zulip issue data to update + :return: the event type """ + # extracts the content into a variable content = issue.get("content", "").lower() - + # checks if the event type is present in the string if "stream created" in content: return "stream created" @@ -293,11 +301,15 @@ def bot_event_type(issue): def notification_bot_event(issue): """ - Docstring for notification_bot_event + Updates the type of event . + This function only updates events notification bot does with no change in username. - :param issue: Description + :param issue: the zulip issue data to update + :return: the event type """ + # extracts the content into a variable content = issue.get("content", "").lower() + # checks if the event type is present in the string if "has marked this topic as resolved" in content: return "topic resolved" @@ -308,11 +320,14 @@ def notification_bot_event(issue): return "topic moved" return "user event" + def bot_event_name_update(issue): """ - Docstring for bot_event_name_update + For events from notification bot. + It finds the user in content string and returns it - :param issue: Description + :param issue: a single zulip-issue data to find the user name for. + :return: returns the user name """ soup = BeautifulSoup(issue["content"], "html.parser") mention = soup.find("span", class_="user-mention") @@ -324,18 +339,22 @@ def bot_event_name_update(issue): def event_type(issue_data): """ - Docstring for event_type - - :param issue_data: Description + Checks if the event is a stream events. + updates the event type and sender details, if sender name is made into notification bot. + :param issue_data: total zulip issue data to check and update the event types. + :return: returns the updated zulip issue data """ for issue in issue_data: if(("stream events" in issue["discusssion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): + # add comments here issue["individual_events"]= bot_event_type(issue) issue["sender_full_name"] = bot_event_name_update(issue) issue["sender_email"] = None else: if("stream events" in issue["discusssion_topic"]): + # add comments here + # rename this "notification_bot_event" issue["individual_events"] = notification_bot_event(issue) issue["individual_events"]= "commented event" @@ -344,10 +363,11 @@ def event_type(issue_data): def update(issue_data): """ - updates values in the issue data structure as per requirement. - :params: issue_data: the issue data to be updated.x + updates values in the issue data as per requirement. + :params: issue_data: the issue data to be updated. :return: returns the issue data. """ + # sends the entirity of the issue data to update discussion id, discussion begin, discussion end and event type issue_data = discussion_id_update(issue_data) issue_data = discussion_begin_end_add(issue_data) issue_data = event_type(issue_data) @@ -525,14 +545,14 @@ def get_user_from_id(idx, buffer_db=user_buffer): def print_to_disk(issues, results_folder): """ - Print issues to file "issues-github.list" in the results folder. + Print issues to file "issues-zulip.list" in the results folder. :param issues: the issues to dump - :param results_folder: the folder where to place "issues-github.list" output file + :param results_folder: the folder where to place "issues-zulip.list" output file """ # construct path to output file - output_file = os.path.join(results_folder, "issues-github.list") + output_file = os.path.join(results_folder, "issues-zulip.list") log.info("Dumping output in file '{}'...".format(output_file)) # construct lines of output From 9d2c7ecf0257ab293334ce7c03bba6617ab04204 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 10:48:54 +0100 Subject: [PATCH 45/79] user dict updated and other issues fixed --- api_data_extraction/zulip_data_extraction.py | 20 +++++++- issue_processing/zulip_issue_processing.py | 52 ++++++++++++++------ 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/api_data_extraction/zulip_data_extraction.py b/api_data_extraction/zulip_data_extraction.py index ac2ce9d..e4e832a 100644 --- a/api_data_extraction/zulip_data_extraction.py +++ b/api_data_extraction/zulip_data_extraction.py @@ -1,8 +1,26 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. +""" +This file is able to extract issue data from Zulip. +""" import zulip import json import time -# ZULIP_CONFIG_FILE = "/Users/ritikahiremath/Downloads/zuliprc.txt" ZULIP_CONFIG_FILE = "zuliprc.txt" client = zulip.Client(config_file=ZULIP_CONFIG_FILE) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index cad61af..866046e 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -19,7 +19,7 @@ # Copyright 2019 by Thomas Bock # Copyright 2020-2021 by Thomas Bock # Copyright 2025 by Maximilian Löffler -# Copyright 2025 by Ritika Hiremath +# Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. """ This file is able to extract Zulip issue data from json files. @@ -73,7 +73,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) + # issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) @@ -226,7 +226,7 @@ def discussion_id_update(issue_data): grouped = {} # groupds each discussion topic together to update the id for item in issue_data: - topic = item["discusssion_topic"] + topic = item["discussion_topic"] if topic not in grouped: grouped[topic] = [] grouped[topic].append(item) @@ -336,8 +336,27 @@ def bot_event_name_update(issue): return mention.get_text(strip=True) return None - -def event_type(issue_data): +def create_user(issue): + """ + Creates user for each issue data. + Classifies name and username based on wthere the name has a space or not. + + :param issue: A single issue data from the zulip issue data + :reutrn: returns a dictionary to update issue["user"] + """ + dict_issue = {} + unclassified_name = issue["sender_full_name"] + if " " in unclassified_name: + dict_issue["name"] = unclassified_name + dict_issue["email"] = issue["sender_email"] + dict_issue["username"] = "" + else: + dict_issue["name"] = "" + dict_issue["email"] = issue["sender_email"] + dict_issue["username"] = unclassified_name + return dict_issue + +def event_type_and_user(issue_data): """ Checks if the event is a stream events. updates the event type and sender details, if sender name is made into notification bot. @@ -346,18 +365,19 @@ def event_type(issue_data): """ for issue in issue_data: - if(("stream events" in issue["discusssion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): - # add comments here + if(("stream events" in issue["discussion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): + # updates name and discussion issue["individual_events"]= bot_event_type(issue) issue["sender_full_name"] = bot_event_name_update(issue) issue["sender_email"] = None else: - if("stream events" in issue["discusssion_topic"]): - # add comments here - # rename this "notification_bot_event" + if("stream events" in issue["discussion_topic"]): + # updates the event type when issue name is proper user name. issue["individual_events"] = notification_bot_event(issue) issue["individual_events"]= "commented event" + # creates user for each issue data. Combines name, email and username into a dictionary. + issue["user"] = create_user(issue) return issue_data @@ -367,10 +387,10 @@ def update(issue_data): :params: issue_data: the issue data to be updated. :return: returns the issue data. """ - # sends the entirity of the issue data to update discussion id, discussion begin, discussion end and event type + # sends the entirity of the issue data to update discussion id, discussion begin, discussion end, event type, and user. issue_data = discussion_id_update(issue_data) issue_data = discussion_begin_end_add(issue_data) - issue_data = event_type(issue_data) + issue_data = event_type_and_user(issue_data) return issue_data @@ -408,7 +428,7 @@ def reformat_issues(issue_data): issue["timestamp"] = format_time(issue["timestamp"]) - issue["type"].append("issue") + issue["type"].append("topic") return issue_data @@ -560,7 +580,7 @@ def print_to_disk(issues, results_folder): for issue in issues: lines.append(( issue["discussion_id"], - issue["discusssion_topic"], + issue["discussion_topic"], json.dumps(issue["type"]), json.dumps([]), json.dumps(issue["resolution"]), @@ -568,8 +588,8 @@ def print_to_disk(issues, results_folder): issue["discussion_end"], json.dumps([]), # components issue["individual_events"], - issue["sender_full_name"], - issue["sender_email"], + issue["user"]["name"], + issue["user"]["email"], issue["timestamp"], json.dumps([]), json.dumps([]) From afd5706166ea81983e553f1c325c0ae00f80fdef Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 11:41:47 +0100 Subject: [PATCH 46/79] rebased and setup_logging import fixed --- issue_processing/issue_processing.py | 2 +- issue_processing/jira_issue_processing.py | 1 - .../old_zulip_issue_processing.py | 488 ------------------ issue_processing/zulip_issue_processing.py | 16 +- 4 files changed, 7 insertions(+), 500 deletions(-) delete mode 100644 issue_processing/old_zulip_issue_processing.py diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 3b04013..222a93c 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -39,7 +39,7 @@ from codeface_utils.dbmanager import DBManager from codeface_utils.util import setup_logging from dateutil import parser as dateparser - +from codeface_utils.util import setup_logging from csv_writer import csv_writer from github_user_utils.github_user_utils import copilot_unified_name diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index 909eb8a..919d872 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -39,7 +39,6 @@ from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager from codeface_utils.util import setup_logging - from csv_writer import csv_writer from jira import JIRA diff --git a/issue_processing/old_zulip_issue_processing.py b/issue_processing/old_zulip_issue_processing.py deleted file mode 100644 index 9a57402..0000000 --- a/issue_processing/old_zulip_issue_processing.py +++ /dev/null @@ -1,488 +0,0 @@ -# new zulip issue processing - -""" -This file is able to extract Zulip issue data from json files. -""" - -import argparse -import httplib -import json -import os -import sys -import urllib -from datetime import datetime, timedelta -from logging import getLogger -import base64 - -import operator -from codeface_utils.cluster.idManager import dbIdManager, csvIdManager -from codeface_utils.configuration import Configuration -from codeface_utils.dbmanager import DBManager -from dateutil import parser as dateparser -from datetime import datetime - -from csv_writer import csv_writer - -# datetime format string -datetime_format = "%Y-%m-%d %H:%M:%S" - -def run(): - # get data from zulip api calls . then format it and apply to codeface extraction - # get all needed paths and arguments for the method call. - parser = argparse.ArgumentParser(prog='codeface-extraction-issues-zulip', description='Codeface extraction') - parser.add_argument('-c', '--config', help="Codeface configuration file", default='codeface.conf') - parser.add_argument('-p', '--project', help="Project configuration file", required=True) - parser.add_argument('resdir', help="Directory to store analysis results in") - - # parse arguments - args = parser.parse_args(sys.argv[1:]) - __codeface_conf, __project_conf = map(os.path.abspath, (args.config, args.project)) - - # create configuration - __conf = Configuration.load(__codeface_conf, __project_conf) - - # get source and results folders - __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) - __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) - - # run processing of issue data: - # 1) load the list of issues - issues = load(__srcdir) - # 2) update missing colums - issues = update(issues) - # 3) re-format the issues - issues = reformat_issues(issues) - # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) - # 6) dump result to disk - print_to_disk(issues, __resdir) - - log.info("Zulip issue processing complete!") - - - -def load(source_folder): - """Load issues from disk. - - :param source_folder: the folder where to find 'issues.json' - :return: the loaded issue data - """ - - srcfile = os.path.join(source_folder, "issues.json") - log.info("Loading Github issues from file '{}'...".format(srcfile)) - - # check if file exists and exit early if not - if not os.path.exists(srcfile): - log.error("Zulip issue file '{}' does not exist! Exiting early...".format(srcfile)) - sys.exit(-1) - - with open(srcfile) as issues_file: - issue_data = json.load(issues_file) - - return issue_data - -#UPDATED -def format_time(time): - """ - Format times from different sources to a consistent time format - - :param time: the time that shall be formatted - :return: the formatted time - """ - - # empty time would be formatted to current date - if time == "" or time is None: - return "" - else: - d = datetime.fromtimestamp(time) - return d.strftime(datetime_format) - - -def subtract_seconds_from_time(time, seconds): - """ - Subtract the specified number of seconds from a date string - - :param time: the date string to subtract the specified seconds from - :param seconds: the number of seconds to subtract from the date string - :return: the date string after subtracting the specified number of seconds - """ - - new_time = datetime.strptime(time, datetime_format) - timedelta(seconds = seconds) - return new_time.strftime(datetime_format) - - -def create_user(name, username, email): - """ - Creates a user object with all needed information - - :param name: the name the user shall have - :param username: the username the user shall have - :param email: the email the user shall have - :return: the created user object - """ - - if name is None: - name = "" - if username is None: - username = "" - if email is None: - email = "" - - user = dict() - user["name"] = name - user["username"] = username - user["email"] = email - - return user - - -def create_deleted_user(): - """ - Creates a user object for a deleted user (ghost user) - - :return: the created user object for a deleted user - """ - - return create_user("Deleted user", "ghost", "ghost@github.com") - - -def lookup_user(user_dict, user): - """ - Alters a user object in the case that name or email are missing by the corresponding name and email - from a given user dictionary - - :param user_dict: the user dictionary - :param user: the user object to lookup in the dictionary - :return: the altered user object in case of a lookup - or the unaltered user object otherwise - """ - - # if user is None, replace it by a deleted user - if user is None: - user = create_deleted_user() - - if (user["name"] == "" or user["name"] is None or - user["email"] is None or user["email"] == ""): - - # lookup user only if username is not None and not empty - if not user["username"] is None and not user["username"] == "": - user = user_dict[user["username"]] - - return user - - -def update_user_dict(user_dict, user): - """ - Adds or updates users to merge GitHub usernames and names and e-mail addresses originating from the git repository - - :param user_dict: the user dictionary - :param user: the user object to add to or update in the dictionary - :return: the updated user dictionary - """ - - # if the given user is None, use the deleted user instead - if user is None: - user = create_deleted_user() - - if not user["username"] in user_dict.keys(): - if not user["username"] is None and not user["username"] == "": - user_dict[user["username"]] = user - else: - user_in_dict = user_dict[user["username"]] - if user_in_dict["name"] is None or user_in_dict["name"] == "": - user_in_dict["name"] = user["name"] - if user_in_dict["email"] is None or user_in_dict["email"] == "": - user_in_dict["email"] = user["email"] - user_dict[user["username"]] = user_in_dict - - return user_dict - -def discussion_id_update(issue_data): - """ - :param issue_data: - """ - # Group timestamps by discussion_topic - grouped = {} - - for item in issue_data: - topic = item["discusssion_topic"] - if topic not in grouped: - grouped[topic] = [] - grouped[topic].append(item) - - # Process each topic group - for topic, messages in grouped.items(): - - # 1. Sort messages by timestamp ASC - messages.sort(key=lambda m: m["timestamp"]) - - # 2. Add sequential discussion_id suffix (#1, #2, #3...) - for idx, msg in enumerate(messages, start=1): - msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' - - return issue_data - -def discussion_begin_end_add(issue_data): - """ - :param issue_data: the issue data where discussion_begin and discussion_end time is to be added - :return: the updated data - """ - # Group timestamps by discussion_topic - discussion_topics = {} - - for item in issue_data: - d_topic = item["discussion_topic"] - ts = item["timestamp"] - - if d_topic not in discussion_topics: - discussion_topics[d_topic] = [] - discussion_topics[d_topic].append(ts) - - # Compute begin and end for each discussion - discussion_bounds = { - d_topic: { - "discussion_begin": min(times), - "discussion_end": max(times) - } - for d_topic, times in discussion_topics.items() - } - - # Add begin/end back to each entry - for item in issue_data: - d_topic = item["discussion_topic"] - item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] - item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] - - return issue_data - -def update(issue_data): - """ - updates values in the issue data structure as per requirement. - :params: issue_data: the issue data to be updated.x - :return: returns the issue data. - """ - # updating id of issues in a particular format. - issue_data = discussion_id_update(issue_data) - # adding discussion begin and end time for each issue data. - issue_data = discussion_begin_end_add(issue_data) - - - return issue_data - -def reformat_issues(issue_data): - """ - Re-arrange issue data structure. - - :param issue_data: the issue data to re-arrange - :return: the re-arranged issue data - """ - - log.info("Re-arranging Github issues...") - - # re-process all issues - for issue in issue_data: - - # empty container for issue types - issue["type"] = [] - - # empty container for issue resolutions - issue["resolution"] = [] - - - # if an issue has no relatedCommits, an empty List gets created - if issue["relatedCommits"] is None: - issue["relatedCommits"] = [] - - # if an issue has no relatedIssues, an empty List gets created - if "relatedIssues" not in issue: - issue["relatedIssues"] = [] - - # add "closed_at" information if not present yet - # if issue["closed_at"] is None: - # issue["closed_at"] = "" - - # parses the creation time in the correct format - # issue["created_at"] = format_time(issue["created_at"]) - - # parses the close time in the correct format - # issue["closed_at"] = format_time(issue["closed_at"]) - - issue["discussion_begin"] = format_time(issue["discussion_begin"]) - - # parses the close time in the correct format - issue["discussion_end"] = format_time(issue["discussion_end"]) - - # checks if the issue is a pull-request or a normal issue and adapts the type - issue["type"].append("issue") - - return issue_data - - - - -def insert_user_data(issues, conf, resdir): - """ - Insert user data into database and update issue data. - In addition, dump username-to-user list to file. - - :param issues: the issues to retrieve user data from - :param conf: the project configuration - :param resdir: the directory in which the username-to-user-list should be dumped - :return: the updated issue data - """ - - log.info("Syncing users with ID service...") - - # create buffer for users (key: user id) - user_buffer = dict() - # create buffer for user ids (key: user string) - user_id_buffer = dict() - # create buffer for usernames (key: username) - username_id_buffer = dict() - # open database connection - dbm = DBManager(conf) - # open ID-service connection - idservice = idManager(dbm, conf) - - def get_user_string(name, email): - if not email or email is None: - return "{name}".format(name=name) - # return "{name} <{name}@default.com>".format(name=name) # for debugging only - else: - return "{name} <{email}>".format(name=name, email=email) - - def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames=username_id_buffer): - username = unicode(user["username"]).encode("utf-8") - - # fix encoding for name and e-mail address - if user["name"] is not None: - name = unicode(user["name"]).encode("utf-8") - else: - name = username - mail = unicode(user["email"]).encode("utf-8") - # construct string for ID service and send query - user_string = get_user_string(name, mail) - - # check buffer to reduce amount of DB queries - if user_string in buffer_db_ids: - log.devinfo("Returning person id for user '{}' from buffer.".format(user_string)) - if username is not None: - buffer_usernames[username] = buffer_db_ids[user_string] - return buffer_db_ids[user_string] - - # get person information from ID service - log.devinfo("Passing user '{}' to ID service.".format(user_string)) - idx = idservice.getPersonID(user_string) - - # add user information to buffer - # user_string = get_user_string(user["name"], user["email"]) # update for - buffer_db_ids[user_string] = idx - - # add id to username buffer - if username is not None: - buffer_usernames[username] = idx - - return idx - - def get_user_from_id(idx, buffer_db=user_buffer): - - # check whether user information is in buffer to reduce amount of DB queries - if idx in buffer_db: - log.devinfo("Returning user '{}' from buffer.".format(idx)) - return buffer_db[idx] - - # get person information from ID service - log.devinfo("Passing user id '{}' to ID service.".format(idx)) - person = idservice.getPersonFromDB(idx) - user = dict() - user["email"] = person["email1"] # column "email1" - user["name"] = person["name"] # column "name" - user["id"] = person["id"] # column "id" - - # add user information to buffer - buffer_db[idx] = user - - return user - - - # check and update database for all occurring users - for issue in issues: - # check database for issue author - issue["user"] = get_id_and_update_user(issue["user"]) - - # check database for event authors - for event in issue["eventsList"]: - event["user"] = get_id_and_update_user(event["user"]) - - # check database for the reference-target user if needed - if event["ref_target"] != "": - event["ref_target"] = get_id_and_update_user(event["ref_target"]) - - # get all users after database updates having been performed - for issue in issues: - # get issue author - issue["user"] = get_user_from_id(issue["user"]) - - # get event authors - for event in issue["eventsList"]: - event["user"] = get_user_from_id(event["user"]) - - # get the reference-target user if needed - if event["ref_target"] != "": - event["ref_target"] = get_user_from_id(event["ref_target"]) - event["event_info_1"] = event["ref_target"]["name"] - event["event_info_2"] = event["ref_target"]["email"] - - # dump username, name, and e-mail to file - lines = [] - for username in username_id_buffer: - user = get_user_from_id(username_id_buffer[username]) - lines.append(( - # username, - user["name"], - user["email"] - )) - - log.info("Dump username list to file...") - username_dump = os.path.join(resdir, "usernames.list") - csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) - - return issues - - -def print_to_disk(issues, results_folder): - """ - Print issues to file "issues-github.list" in the results folder. - - :param issues: the issues to dump - :param results_folder: the folder where to place "issues-github.list" output file - """ - - # construct path to output file - output_file = os.path.join(results_folder, "issues-github.list") - log.info("Dumping output in file '{}'...".format(output_file)) - - # construct lines of output - lines = [] - for issue in issues: - for event in issue["eventsList"]: - lines.append(( - issue["number"], - issue["title"], - json.dumps(issue["type"]), - issue["state_new"], - json.dumps(issue["resolution"]), - issue["created_at"], - issue["closed_at"], - json.dumps([]), # components - event["event"], - event["user"]["name"], - event["user"]["email"], - event["created_at"], - event["event_info_1"], - json.dumps(event["event_info_2"]) - )) - - # write to output file - csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) - diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 866046e..39e73ae 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -18,6 +18,7 @@ # Copyright 2018-2019 by Anselm Fehnker # Copyright 2019 by Thomas Bock # Copyright 2020-2021 by Thomas Bock +# Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. @@ -35,12 +36,14 @@ from codeface_utils.cluster.idManager import dbIdManager, csvIdManager from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager +from codeface_utils.util import setup_logging from dateutil import parser as dateparser from bs4 import BeautifulSoup from csv_writer import csv_writer - +# create logger +setup_logging() log = getLogger(__name__) # datetime format string @@ -65,8 +68,10 @@ def run(): __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) # run processing of issue data: + print(log.name) # 1) load the list of issues issues = load(__srcdir) + log.info("Source file loaded") # 2) re-format the issues # 2) update missing colums issues = update(issues) @@ -76,7 +81,6 @@ def run(): # issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) - log.info("Zulip issue processing complete!") @@ -413,14 +417,6 @@ def reformat_issues(issue_data): # empty container for issue resolutions issue["resolution"] = [] - # if an issue has no relatedCommits, an empty List gets created - if issue["relatedCommits"] is None: - issue["relatedCommits"] = [] - - # if an issue has no relatedIssues, an empty List gets created - if "relatedIssues" not in issue: - issue["relatedIssues"] = [] - issue["discussion_begin"] = format_time(issue["discussion_begin"]) # parses the close time in the correct format From a1cfe3137c2bdbf8b6da0298637a358182972359 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 11:46:02 +0100 Subject: [PATCH 47/79] updated copyright for logging import --- issue_processing/issue_processing.py | 1 + issue_processing/jira_issue_processing.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 222a93c..da0169a 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -21,6 +21,7 @@ # Copyright 2025-2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler +# Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. """ This file is able to extract Github issue data from json files. diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index 919d872..ed101c4 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -19,7 +19,11 @@ # Copyright 2020-2021 by Thomas Bock # Copyright 2026 by Thomas Bock # Copyright 2023, 2025 by Maximilian Löffler +<<<<<<< HEAD # Copyright 2025-2026 by Leo Sendelbach +======= +# Copyright 2025-2026 by Ritika Hiremath +>>>>>>> 32ac114 (updated copyright for logging import) # All Rights Reserved. """ This file is able to extract Jira issue data from xml files. From 504f4cf2125082060e6e38c030a70135c2d12a6a Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 11:49:13 +0100 Subject: [PATCH 48/79] uncommented insert_user_data --- issue_processing/zulip_issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 39e73ae..18f4255 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -78,7 +78,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - # issues = insert_user_data(issues, __conf, __resdir) + issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) log.info("Zulip issue processing complete!") From 634a12d253961d80be41d0cfef49b143b2ef01b6 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 12:29:48 +0100 Subject: [PATCH 49/79] additional logging added --- issue_processing/zulip_issue_processing.py | 31 +++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 18f4255..4791090 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -78,7 +78,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) + # issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) log.info("Zulip issue processing complete!") @@ -231,8 +231,12 @@ def discussion_id_update(issue_data): # groupds each discussion topic together to update the id for item in issue_data: topic = item["discussion_topic"] + if not topic: + log.warning("Missing discussion_topic in item:" + str(item)) + continue if topic not in grouped: grouped[topic] = [] + log.debug("New topic group created:" + str(topic)) grouped[topic].append(item) # Updates the disucssion id here. @@ -241,6 +245,7 @@ def discussion_id_update(issue_data): for idx, msg in enumerate(messages, start=1): msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' + log.info("Finished updating discussion id for all issue data") return issue_data def discussion_begin_end_add(issue_data): @@ -258,8 +263,12 @@ def discussion_begin_end_add(issue_data): if d_topic not in discussion_topics: discussion_topics[d_topic] = [] + log.debug("New discussion topic found:" + str(d_topic)) discussion_topics[d_topic].append(ts) + if not discussion_topics: + log.warning("No discussion topics were grouped!") + discussion_bounds = { d_topic: { "discussion_begin": min(times), @@ -268,11 +277,16 @@ def discussion_begin_end_add(issue_data): for d_topic, times in discussion_topics.items() } + # for d_topic, bounds in discussion_bounds.items(): + # log.debug("Topic" + str(d_topic) + "->" + str(bounds)) + for item in issue_data: d_topic = item["discussion_topic"] item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] + log.info("Discussion bounds updated for "+ str(item)) + log.info("Finished updating discussion bounds") return issue_data def bot_event_type(issue): @@ -340,6 +354,7 @@ def bot_event_name_update(issue): return mention.get_text(strip=True) return None + def create_user(issue): """ Creates user for each issue data. @@ -348,16 +363,22 @@ def create_user(issue): :param issue: A single issue data from the zulip issue data :reutrn: returns a dictionary to update issue["user"] """ + log.debug("Creating user for issue" + str(issue["discussion_id"])) + dict_issue = {} unclassified_name = issue["sender_full_name"] if " " in unclassified_name: + log.debug("Detected full name:" + str(unclassified_name)) dict_issue["name"] = unclassified_name - dict_issue["email"] = issue["sender_email"] dict_issue["username"] = "" else: + log.debug("Detected username only:" + str(unclassified_name)) dict_issue["name"] = "" - dict_issue["email"] = issue["sender_email"] dict_issue["username"] = unclassified_name + + dict_issue["email"] = issue["sender_email"] + + log.info("New User dict created for" + str(issue["discussion_topic"]) + "with" + str(issue["discussion_id"])) return dict_issue def event_type_and_user(issue_data): @@ -370,12 +391,14 @@ def event_type_and_user(issue_data): for issue in issue_data: if(("stream events" in issue["discussion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): + log.debug("Bot stream event detected") # updates name and discussion issue["individual_events"]= bot_event_type(issue) issue["sender_full_name"] = bot_event_name_update(issue) issue["sender_email"] = None else: if("stream events" in issue["discussion_topic"]): + log.debug("User stream event detected") # updates the event type when issue name is proper user name. issue["individual_events"] = notification_bot_event(issue) @@ -383,6 +406,7 @@ def event_type_and_user(issue_data): # creates user for each issue data. Combines name, email and username into a dictionary. issue["user"] = create_user(issue) + log.info("Finished updating event types and users") return issue_data def update(issue_data): @@ -396,6 +420,7 @@ def update(issue_data): issue_data = discussion_begin_end_add(issue_data) issue_data = event_type_and_user(issue_data) + log.info("Issue data updated ...") return issue_data def reformat_issues(issue_data): From 6fa4f1705f5a891ecdec66e6ff8fe1bdfb485fcc Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 16 Feb 2026 12:32:25 +0100 Subject: [PATCH 50/79] uncommented insert_user_data --- issue_processing/zulip_issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 4791090..ef3af59 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -78,7 +78,7 @@ def run(): # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list - # issues = insert_user_data(issues, __conf, __resdir) + issues = insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) log.info("Zulip issue processing complete!") From 78c9adbf91655c89922601a0dc614e229e7b1b9a Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 14:19:47 +0100 Subject: [PATCH 51/79] processing userlist featuer updated and log statements reduced --- issue_processing/zulip_issue_processing.py | 148 +++++++++++++++------ 1 file changed, 111 insertions(+), 37 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index ef3af59..f79445f 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -66,15 +66,15 @@ def run(): # get source and results folders __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) - + __userdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_usernames")) # run processing of issue data: - print(log.name) # 1) load the list of issues issues = load(__srcdir) log.info("Source file loaded") + users = load_users(__userdir) # 2) re-format the issues # 2) update missing colums - issues = update(issues) + issues = update(issues, users) # 3) re-format the issues issues = reformat_issues(issues) # 5) update user data with Codeface database and dump username-to-name/e-mail list @@ -104,6 +104,48 @@ def load(source_folder): return issue_data +def load_users(source_folder): + """Load users list from disk. + + :param source_folder: the folder where to find 'usernames.list' + :return: the loaded zulip data + """ + + srcfile = os.path.join(source_folder, "usernames.list") + log.info("Loading users data from file '{}'...".format(srcfile)) + + # check if file exists and exit early if not + if not os.path.exists(srcfile): + log.error("Users data file '{}' does not exist! Exiting early...".format(srcfile)) + sys.exit(-1) + + users = {} + # cleanes and opens the source file + with open(srcfile, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + + parts = line.split(";") + if len(parts) != 3: + log.warning(f"Skipping malformed line: {line}") + continue + + username, name, email = [p.strip('"') for p in parts] + + # creates access to users data through username unless it is None + if username != "None": + users[username] = {"name": name, "email": email} + + # creates access to users data through name + users[name] = { + "username": "" if username == "None" else username, + "email": email, + } + + log.info(f"Loaded {len(users)} users") + return users def format_time(time): """ @@ -284,7 +326,7 @@ def discussion_begin_end_add(issue_data): d_topic = item["discussion_topic"] item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] - log.info("Discussion bounds updated for "+ str(item)) + log.debug("Discussion bounds updated for "+ str(item)) log.info("Finished updating discussion bounds") return issue_data @@ -355,7 +397,23 @@ def bot_event_name_update(issue): return None -def create_user(issue): +# def parse_usernames(lines): + +# users = {} + +# for line in lines: +# parts = line.strip().split(";") +# username, name, email = [p.strip('"') for p in parts] + +# if username != "None": +# users[username] = {"name": name, "email": email} + +# users[name] = {"username": username if username != "None" else "", +# "email": email} + +# return users + +def create_update_user(issue, users): """ Creates user for each issue data. Classifies name and username based on wthere the name has a space or not. @@ -363,25 +421,34 @@ def create_user(issue): :param issue: A single issue data from the zulip issue data :reutrn: returns a dictionary to update issue["user"] """ - log.debug("Creating user for issue" + str(issue["discussion_id"])) + log.debug("Creating user for issue " + str(issue["discussion_id"])) dict_issue = {} - unclassified_name = issue["sender_full_name"] - if " " in unclassified_name: - log.debug("Detected full name:" + str(unclassified_name)) - dict_issue["name"] = unclassified_name - dict_issue["username"] = "" + sender = issue["sender_full_name"] + + # check if present in users list + if sender in users: + info = users[sender] + + dict_issue["name"] = info.get("name", sender) + dict_issue["username"] = info.get("username", sender) + dict_issue["email"] = info.get("email", issue["sender_email"]) + else: - log.debug("Detected username only:" + str(unclassified_name)) - dict_issue["name"] = "" - dict_issue["username"] = unclassified_name - - dict_issue["email"] = issue["sender_email"] + # fallback + if " " in sender: + dict_issue["name"] = sender + dict_issue["username"] = "" + else: + dict_issue["name"] = "" + dict_issue["username"] = sender - log.info("New User dict created for" + str(issue["discussion_topic"]) + "with" + str(issue["discussion_id"])) + dict_issue["email"] = issue["sender_email"] + + # log.debug("New User dict created for" + str(issue["discussion_topic"]) + "with" + str(issue["discussion_id"])) return dict_issue -def event_type_and_user(issue_data): +def event_type_and_user(issue_data, username): """ Checks if the event is a stream events. updates the event type and sender details, if sender name is made into notification bot. @@ -391,34 +458,41 @@ def event_type_and_user(issue_data): for issue in issue_data: if(("stream events" in issue["discussion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): - log.debug("Bot stream event detected") + # log.debug("Bot stream event detected") # updates name and discussion issue["individual_events"]= bot_event_type(issue) issue["sender_full_name"] = bot_event_name_update(issue) - issue["sender_email"] = None + soup = BeautifulSoup(issue["content"], "html.parser") + span_data = soup.find("span", class_="user-mention") + if span_data: + user_id = span_data.get("data-user-id") + issue["sender_email"] = user_id+"@zulipchat.com" + else: if("stream events" in issue["discussion_topic"]): - log.debug("User stream event detected") + # log.debug("User stream event detected") # updates the event type when issue name is proper user name. issue["individual_events"] = notification_bot_event(issue) issue["individual_events"]= "commented event" # creates user for each issue data. Combines name, email and username into a dictionary. - issue["user"] = create_user(issue) + issue["user"] = create_update_user(issue, username) log.info("Finished updating event types and users") return issue_data -def update(issue_data): +def update(issue_data, users): """ updates values in the issue data as per requirement. :params: issue_data: the issue data to be updated. + :params: usernames: existing list of user data :return: returns the issue data. """ + # sends the entirity of the issue data to update discussion id, discussion begin, discussion end, event type, and user. issue_data = discussion_id_update(issue_data) issue_data = discussion_begin_end_add(issue_data) - issue_data = event_type_and_user(issue_data) + issue_data = event_type_and_user(issue_data,users) log.info("Issue data updated ...") return issue_data @@ -546,12 +620,12 @@ def get_user_from_id(idx, buffer_db=user_buffer): issue["user"] = get_id_and_update_user(issue["user"]) # check database for event authors - for event in issue["eventsList"]: - event["user"] = get_id_and_update_user(event["user"]) + # for event in issue["eventsList"]: + # event["user"] = get_id_and_update_user(event["user"]) - # check database for the reference-target user if needed - if event["ref_target"] != "": - event["ref_target"] = get_id_and_update_user(event["ref_target"]) + # # check database for the reference-target user if needed + # if event["ref_target"] != "": + # event["ref_target"] = get_id_and_update_user(event["ref_target"]) # get all users after database updates having been performed for issue in issues: @@ -559,14 +633,14 @@ def get_user_from_id(idx, buffer_db=user_buffer): issue["user"] = get_user_from_id(issue["user"]) # get event authors - for event in issue["eventsList"]: - event["user"] = get_user_from_id(event["user"]) - - # get the reference-target user if needed - if event["ref_target"] != "": - event["ref_target"] = get_user_from_id(event["ref_target"]) - event["event_info_1"] = event["ref_target"]["name"] - event["event_info_2"] = event["ref_target"]["email"] + # for event in issue["eventsList"]: + # event["user"] = get_user_from_id(event["user"]) + + # # get the reference-target user if needed + # if event["ref_target"] != "": + # event["ref_target"] = get_user_from_id(event["ref_target"]) + # event["event_info_1"] = event["ref_target"]["name"] + # event["event_info_2"] = event["ref_target"]["email"] # dump username, name, and e-mail to file lines = [] From 2e3d73482f1a34ab5b730afa108e72f29d09f2ed Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 15:55:03 +0100 Subject: [PATCH 52/79] name, username issue fixed --- issue_processing/zulip_issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index f79445f..b8a27f8 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -440,7 +440,7 @@ def create_update_user(issue, users): dict_issue["name"] = sender dict_issue["username"] = "" else: - dict_issue["name"] = "" + dict_issue["name"] = sender dict_issue["username"] = sender dict_issue["email"] = issue["sender_email"] From 9d62f2a0af32c3c82011cf3d65ce974435871daa Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 16:11:01 +0100 Subject: [PATCH 53/79] removed commented code --- issue_processing/zulip_issue_processing.py | 34 ---------------------- 1 file changed, 34 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index b8a27f8..385646c 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -397,22 +397,6 @@ def bot_event_name_update(issue): return None -# def parse_usernames(lines): - -# users = {} - -# for line in lines: -# parts = line.strip().split(";") -# username, name, email = [p.strip('"') for p in parts] - -# if username != "None": -# users[username] = {"name": name, "email": email} - -# users[name] = {"username": username if username != "None" else "", -# "email": email} - -# return users - def create_update_user(issue, users): """ Creates user for each issue data. @@ -619,29 +603,11 @@ def get_user_from_id(idx, buffer_db=user_buffer): # check database for issue author issue["user"] = get_id_and_update_user(issue["user"]) - # check database for event authors - # for event in issue["eventsList"]: - # event["user"] = get_id_and_update_user(event["user"]) - - # # check database for the reference-target user if needed - # if event["ref_target"] != "": - # event["ref_target"] = get_id_and_update_user(event["ref_target"]) - # get all users after database updates having been performed for issue in issues: # get issue author issue["user"] = get_user_from_id(issue["user"]) - # get event authors - # for event in issue["eventsList"]: - # event["user"] = get_user_from_id(event["user"]) - - # # get the reference-target user if needed - # if event["ref_target"] != "": - # event["ref_target"] = get_user_from_id(event["ref_target"]) - # event["event_info_1"] = event["ref_target"]["name"] - # event["event_info_2"] = event["ref_target"]["email"] - # dump username, name, and e-mail to file lines = [] for username in username_id_buffer: From 1db77de982161f9a0a29e80964c6a3b5fbbe72b7 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 16:27:28 +0100 Subject: [PATCH 54/79] username.list input made optional --- issue_processing/zulip_issue_processing.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 385646c..8649382 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -66,7 +66,7 @@ def run(): # get source and results folders __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) - __userdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_usernames")) + __userdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) # run processing of issue data: # 1) load the list of issues issues = load(__srcdir) @@ -105,7 +105,7 @@ def load(source_folder): return issue_data def load_users(source_folder): - """Load users list from disk. + """Load users list from disk if it exists. :param source_folder: the folder where to find 'usernames.list' :return: the loaded zulip data @@ -116,9 +116,9 @@ def load_users(source_folder): # check if file exists and exit early if not if not os.path.exists(srcfile): - log.error("Users data file '{}' does not exist! Exiting early...".format(srcfile)) - sys.exit(-1) - + log.error("Users data file '{}' does not exist! Continuing without...".format(srcfile)) + return {} + users = {} # cleanes and opens the source file with open(srcfile, encoding="utf-8") as f: From b4f530728170d934414fd7397a16c54b89a09a13 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 17:04:29 +0100 Subject: [PATCH 55/79] comments added order changed --- issue_processing/zulip_issue_processing.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 8649382..96fc0dd 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -194,8 +194,8 @@ def create_user(name, username, email): email = "" user = dict() - user["name"] = name user["username"] = username + user["name"] = name user["email"] = email return user @@ -414,18 +414,21 @@ def create_update_user(issue, users): if sender in users: info = users[sender] - dict_issue["name"] = info.get("name", sender) dict_issue["username"] = info.get("username", sender) + dict_issue["name"] = info.get("name", sender) dict_issue["email"] = info.get("email", issue["sender_email"]) else: # fallback + # checks if there is a space in the string sender. if " " in sender: - dict_issue["name"] = sender dict_issue["username"] = "" - else: dict_issue["name"] = sender + + # if it does not find a name then updates both with sa + else: dict_issue["username"] = sender + dict_issue["name"] = sender dict_issue["email"] = issue["sender_email"] @@ -639,6 +642,7 @@ def print_to_disk(issues, results_folder): # construct lines of output lines = [] for issue in issues: + print(issue["user"]) lines.append(( issue["discussion_id"], issue["discussion_topic"], From 928d36f09dcfe39ae18da172f7d425df67386354 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 2 Mar 2026 17:08:10 +0100 Subject: [PATCH 56/79] additional print statement commented --- issue_processing/zulip_issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 96fc0dd..93bc995 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -642,7 +642,7 @@ def print_to_disk(issues, results_folder): # construct lines of output lines = [] for issue in issues: - print(issue["user"]) + # print(issue["user"]) lines.append(( issue["discussion_id"], issue["discussion_topic"], From c63f65f7b62d1c5a5436d4a3778069dfd4596850 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Sun, 22 Mar 2026 22:42:28 +0100 Subject: [PATCH 57/79] typos fixed in zulip issue processing --- issue_processing/zulip_issue_processing.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 93bc995..4ba6c1c 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -72,8 +72,7 @@ def run(): issues = load(__srcdir) log.info("Source file loaded") users = load_users(__userdir) - # 2) re-format the issues - # 2) update missing colums + # 2) update missing columns issues = update(issues, users) # 3) re-format the issues issues = reformat_issues(issues) @@ -120,7 +119,7 @@ def load_users(source_folder): return {} users = {} - # cleanes and opens the source file + # cleans and opens the source file with open(srcfile, encoding="utf-8") as f: for line in f: line = line.strip() @@ -270,7 +269,7 @@ def discussion_id_update(issue_data): : return: The updated issue data with the id updated. """ grouped = {} - # groupds each discussion topic together to update the id + # groups each discussion topic together to update the id for item in issue_data: topic = item["discussion_topic"] if not topic: @@ -281,7 +280,7 @@ def discussion_id_update(issue_data): log.debug("New topic group created:" + str(topic)) grouped[topic].append(item) - # Updates the disucssion id here. + # Updates the discussion id here. for topic, messages in grouped.items(): messages.sort(key=lambda m: m["timestamp"]) for idx, msg in enumerate(messages, start=1): @@ -400,7 +399,7 @@ def bot_event_name_update(issue): def create_update_user(issue, users): """ Creates user for each issue data. - Classifies name and username based on wthere the name has a space or not. + Classifies name and username based on whethre the name has a space or not. :param issue: A single issue data from the zulip issue data :reutrn: returns a dictionary to update issue["user"] @@ -444,7 +443,7 @@ def event_type_and_user(issue_data, username): """ for issue in issue_data: - if(("stream events" in issue["discussion_topic"]) & (issue["sender_full_name"] == "Notification Bot")): + if(("stream events" in issue["discussion_topic"]) and (issue["sender_full_name"] == "Notification Bot")): # log.debug("Bot stream event detected") # updates name and discussion issue["individual_events"]= bot_event_type(issue) @@ -476,7 +475,7 @@ def update(issue_data, users): :return: returns the issue data. """ - # sends the entirity of the issue data to update discussion id, discussion begin, discussion end, event type, and user. + # sends the entirety of the issue data to update discussion id, discussion begin, discussion end, event type, and user. issue_data = discussion_id_update(issue_data) issue_data = discussion_begin_end_add(issue_data) issue_data = event_type_and_user(issue_data,users) From fa8fc9a21c78fd239729d59a986bf4f481efc9b6 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 23 Mar 2026 21:58:57 +0100 Subject: [PATCH 58/79] zulip-data-extraction updated --- zulip_data_extraction/__init__.py | 1 + .../zulip_data_extraction.py | 21 +++++++++++++------ zulip_data_extraction/zuliprc.txt | 4 ++++ 3 files changed, 20 insertions(+), 6 deletions(-) create mode 100644 zulip_data_extraction/__init__.py rename {api_data_extraction => zulip_data_extraction}/zulip_data_extraction.py (85%) create mode 100644 zulip_data_extraction/zuliprc.txt diff --git a/zulip_data_extraction/__init__.py b/zulip_data_extraction/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/zulip_data_extraction/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/api_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py similarity index 85% rename from api_data_extraction/zulip_data_extraction.py rename to zulip_data_extraction/zulip_data_extraction.py index e4e832a..c649ccb 100644 --- a/api_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -20,7 +20,16 @@ import zulip import json import time +from codeface_utils.util import setup_logging +from logging import getLogger + +# create logger +setup_logging() +log = getLogger(__name__) + +# Log in to https://rust-lang.zulipchat.com and in the API Key present in personal settings, download zuliprc.txt +# Template of zuliprc.txt is present in this directory. ZULIP_CONFIG_FILE = "zuliprc.txt" client = zulip.Client(config_file=ZULIP_CONFIG_FILE) @@ -39,10 +48,10 @@ def safe_get_topics(stream_id): return [t["name"] for t in resp["topics"]] elif resp["result"] == "error" and resp.get("code") == "RATE_LIMIT_HIT": retry = int(resp.get("retry_after", 5)) - print(f"Rate limit hit, retrying in {retry}s...") + log.info("Rate limit hit, retrying in '{}'s...".format(retry)) time.sleep(retry) else: - print("Error fetching topics:", resp) + log.error("Error fetching topics: '{}'", resp) return [] @@ -57,7 +66,7 @@ def topics_extraction(): for i, s in enumerate(streams, 1): stream_name = s["name"] - print(f"[{i}/{len(streams)}] Getting topics for: {stream_name}") + log.debug(f"[{i}/{len(streams)}] Getting topics for: {stream_name}") topics = safe_get_topics(s["stream_id"]) data[stream_name] = { @@ -69,7 +78,7 @@ def topics_extraction(): with open("zulip_streams_and_topics.json", "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) - print("Saved zulip_streams_and_topics.json") + log.info("Saved zulip_streams_and_topics.json") return data @@ -131,7 +140,7 @@ def messages_extraction_for_each_stream(streams_with_topics): topics = info["topics"] for topic in topics: - print(f"\n Fetching all messages for stream: {stream_name} and topic: {topic}") + log.debug(f"\n Fetching all messages for stream: {stream_name} and topic: {topic}") msgs = fetch_all_messages_for_stream(stream_name,topic) @@ -151,7 +160,7 @@ def messages_extraction_for_each_stream(streams_with_topics): with open(output_file, "w") as f: json.dump(final_output, f, indent=2) - print(f"\n Saved ALL stream messages to: {output_file}") + log.info("\n Saved all stream messages to: '{}'".format(output_file)) if __name__ == "__main__": streams_and_topics = topics_extraction() diff --git a/zulip_data_extraction/zuliprc.txt b/zulip_data_extraction/zuliprc.txt new file mode 100644 index 0000000..37d2142 --- /dev/null +++ b/zulip_data_extraction/zuliprc.txt @@ -0,0 +1,4 @@ +[api] +email= +key= +site=https://rust-lang.zulipchat.com From 5c0904a8f3319e25af3ad3d77a2c5588d24cec8b Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 25 Mar 2026 16:25:08 +0100 Subject: [PATCH 59/79] complete extraction files --- run-zulip-data-extraction.py | 4 ++++ zulip_data_extraction/zulip_data_extraction.py | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) create mode 100644 run-zulip-data-extraction.py diff --git a/run-zulip-data-extraction.py b/run-zulip-data-extraction.py new file mode 100644 index 0000000..7bda412 --- /dev/null +++ b/run-zulip-data-extraction.py @@ -0,0 +1,4 @@ +from zulip_data_extraction import zulip_data_extraction as scrapping + +scrapping.run() + diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py index c649ccb..bfa36ac 100644 --- a/zulip_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -29,8 +29,9 @@ log = getLogger(__name__) # Log in to https://rust-lang.zulipchat.com and in the API Key present in personal settings, download zuliprc.txt -# Template of zuliprc.txt is present in this directory. -ZULIP_CONFIG_FILE = "zuliprc.txt" +# Template of zuliprc.txt is present in this directory. +# Update the location of zuliprc.txt file in the bellow string +ZULIP_CONFIG_FILE = "/Users/ritikahiremath/Downloads/zuliprc.txt" client = zulip.Client(config_file=ZULIP_CONFIG_FILE) @@ -162,6 +163,7 @@ def messages_extraction_for_each_stream(streams_with_topics): log.info("\n Saved all stream messages to: '{}'".format(output_file)) -if __name__ == "__main__": +def run(): + log.info("Starting Zulip data extraction") streams_and_topics = topics_extraction() messages_extraction_for_each_stream(streams_and_topics) From a31b07ac00820bda8e38451fa2ad83931e9d5df9 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 2 Apr 2026 10:15:25 +0200 Subject: [PATCH 60/79] zulip zuliprc.txt file parser updated --- .../zulip_data_extraction.py | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py index bfa36ac..a2b0234 100644 --- a/zulip_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -19,7 +19,10 @@ """ import zulip import json +import argparse import time +import os +import sys from codeface_utils.util import setup_logging from logging import getLogger @@ -28,11 +31,25 @@ setup_logging() log = getLogger(__name__) -# Log in to https://rust-lang.zulipchat.com and in the API Key present in personal settings, download zuliprc.txt +# Log in to https://rust-lang.zulipchat.com, go to Personal Settings → API key, and download the .zuliprc file. # Template of zuliprc.txt is present in this directory. -# Update the location of zuliprc.txt file in the bellow string -ZULIP_CONFIG_FILE = "/Users/ritikahiremath/Downloads/zuliprc.txt" -client = zulip.Client(config_file=ZULIP_CONFIG_FILE) + +# The location of zuliprc.txt file +parser = argparse.ArgumentParser() +parser.add_argument("--zulip-config", default=None, help="Path to zuliprc config file") +args = parser.parse_args(sys.argv[1:]) + +# Resolve path +if args.zulip_config: + config_path = args.zulip_config +else: + config_path = os.path.join(os.path.dirname(__file__), "zuliprc") + +# Raise error if file not found +if not os.path.exists(config_path): + raise FileNotFoundError(f"Zulip config file not found at: {config_path}") + +client = zulip.Client(config_file= config_path) def safe_get_topics(stream_id): From 94c79006e577aae85efae2a2938ac868dee1e71f Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 2 Apr 2026 10:17:45 +0200 Subject: [PATCH 61/79] zulip zuliprc.txt comment updated --- zulip_data_extraction/zulip_data_extraction.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py index a2b0234..b8e2114 100644 --- a/zulip_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -31,7 +31,7 @@ setup_logging() log = getLogger(__name__) -# Log in to https://rust-lang.zulipchat.com, go to Personal Settings → API key, and download the .zuliprc file. +# Log in to https://rust-lang.zulipchat.com, go to Personal Settings -> Account&Privacy -> API key, and download the .zuliprc file or copy the API key and utilize the template. # Template of zuliprc.txt is present in this directory. # The location of zuliprc.txt file From 2c2bd09925078960648e68c18d1e2329a3602456 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 2 Apr 2026 10:18:20 +0200 Subject: [PATCH 62/79] copyright updated --- run-zulip-data-extraction.py | 16 ++++++++++++++++ run-zulip-issues.py | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/run-zulip-data-extraction.py b/run-zulip-data-extraction.py index 7bda412..b3f4e8d 100644 --- a/run-zulip-data-extraction.py +++ b/run-zulip-data-extraction.py @@ -1,3 +1,19 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. from zulip_data_extraction import zulip_data_extraction as scrapping scrapping.run() diff --git a/run-zulip-issues.py b/run-zulip-issues.py index d9a44c1..98f0725 100644 --- a/run-zulip-issues.py +++ b/run-zulip-issues.py @@ -1,3 +1,19 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. import issue_processing.zulip_issue_processing as zulip_issues zulip_issues.run() From a1af2a1bd90c00e0e4bb4aaf0296d8352e76bd21 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Fri, 3 Apr 2026 16:17:37 +0200 Subject: [PATCH 63/79] typo fixed --- run-zulip-data-extraction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run-zulip-data-extraction.py b/run-zulip-data-extraction.py index b3f4e8d..9827220 100644 --- a/run-zulip-data-extraction.py +++ b/run-zulip-data-extraction.py @@ -14,7 +14,7 @@ # # Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. -from zulip_data_extraction import zulip_data_extraction as scrapping +from zulip_data_extraction import zulip_data_extraction as scraping -scrapping.run() +scraping.run() From aef5d99fd707944fbb3270e4f8beb55486bc2ca1 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Sun, 3 May 2026 14:40:47 +0200 Subject: [PATCH 64/79] combine issue feature completed --- combine_issues/__init__.py | 1 + combine_issues/combine_issues.py | 98 ++++++++++++++++++++++++++++++++ run-combine-issues.py | 19 +++++++ 3 files changed, 118 insertions(+) create mode 100644 combine_issues/__init__.py create mode 100644 combine_issues/combine_issues.py create mode 100644 run-combine-issues.py diff --git a/combine_issues/__init__.py b/combine_issues/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/combine_issues/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/combine_issues/combine_issues.py b/combine_issues/combine_issues.py new file mode 100644 index 0000000..12c7977 --- /dev/null +++ b/combine_issues/combine_issues.py @@ -0,0 +1,98 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Ritika Hiremath +# All Rights Reserved. +""" +This file merges different issues_github.list files from different projects. +""" +# coding=utf-8 +import os +import csv +import argparse +from pathlib import Path +from logging import getLogger +from codeface_utils.util import setup_logging + +# create logger +setup_logging() +log = getLogger(__name__) + + +def run(): + parser = argparse.ArgumentParser(description="Merge issues-github.list files") + parser.add_argument( "--resdir", required=True, help="Path to data/results/threemonth/" ) + parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. vue_proximity keras_proximity" ) + parser.add_argument( "--output", default="merged_issues.list", help="Output CSV file path (default: merged_issues.list)" ) + args = parser.parse_args() + + # extract issues + all_issues = extract_issues(args.projects, args.resdir) + # merge and update the issue content + merged = merge_issues(all_issues) + # save merged issues + save_merged(merged, args.output) + log.info("Issues successfully merged!") + +def extract_issues(project_list, threemonth_dir): + """ + Extracts each issues-github.list data from each project and appends to all issues + """ + all_issues = {} + for project in project_list: + # Matches the actual path for data: threemonth//proximity/issues-github.list + issues_file = Path(threemonth_dir) / project / "proximity" / "issues-github.list" + if not issues_file.exists(): + log.warning(f"File not found: {issues_file}") + continue + + with issues_file.open(newline="", encoding="utf-8") as f: + reader = csv.reader(f, delimiter=";") + rows = [row for row in reader] + all_issues[project] = rows + log.info(f"Loaded {len(rows)} rows from '{project}'") + return all_issues + +def merge_issues(all_issues): + """ + All issues are taken and corrected to the required format + """ + merged = [] + for project, rows in all_issues.items(): + short_name = project.replace("_proximity","") + for row in rows: + if not row: + continue + new_row = row.copy() + # Updating firts row: 1 -> keras-1 + new_row[0] = f"{short_name}_{new_row[0]}" + + # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> keras-3885 + last_col = new_row[13].strip().strip('"') + issue_num = new_row[12].strip().strip('"') + + if last_col.lower() == "issue" and issue_num.isdigit(): + new_row[12] = f"{short_name}-{issue_num}" + merged.append(new_row) + log.info(f"Total merged rows: {len(merged)}") + return merged + +def save_merged(merged_rows, output_path): + """ + Saves the file with the updated contents + """ + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL) + writer.writerows(merged_rows) + log.info(f"Saved to {output_path}") diff --git a/run-combine-issues.py b/run-combine-issues.py new file mode 100644 index 0000000..5518f7f --- /dev/null +++ b/run-combine-issues.py @@ -0,0 +1,19 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Ritika Hiremath +# All Rights Reserved. +from combine_issues import combine_issues + +combine_issues.run() From 1ff1b8fab47b66a39bea8ee6fbc9b9d53c21ccf8 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 7 May 2026 10:01:28 +0200 Subject: [PATCH 65/79] os path fixed --- combine_issues/combine_issues.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/combine_issues/combine_issues.py b/combine_issues/combine_issues.py index 12c7977..7363a03 100644 --- a/combine_issues/combine_issues.py +++ b/combine_issues/combine_issues.py @@ -34,7 +34,7 @@ def run(): parser = argparse.ArgumentParser(description="Merge issues-github.list files") parser.add_argument( "--resdir", required=True, help="Path to data/results/threemonth/" ) parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. vue_proximity keras_proximity" ) - parser.add_argument( "--output", default="merged_issues.list", help="Output CSV file path (default: merged_issues.list)" ) + parser.add_argument( "--output", required= True, help="Custom output directory name" ) args = parser.parse_args() # extract issues @@ -42,7 +42,7 @@ def run(): # merge and update the issue content merged = merge_issues(all_issues) # save merged issues - save_merged(merged, args.output) + save_merged(merged, args.resdir, args.projects, args.output) log.info("Issues successfully merged!") def extract_issues(project_list, threemonth_dir): @@ -52,12 +52,12 @@ def extract_issues(project_list, threemonth_dir): all_issues = {} for project in project_list: # Matches the actual path for data: threemonth//proximity/issues-github.list - issues_file = Path(threemonth_dir) / project / "proximity" / "issues-github.list" - if not issues_file.exists(): + issues_file = os.path.join(threemonth_dir, project, "proximity", "issues-github.list") + if not os.path.exists(issues_file): log.warning(f"File not found: {issues_file}") continue - with issues_file.open(newline="", encoding="utf-8") as f: + with open(issues_file, newline="", encoding="utf-8") as f: reader = csv.reader(f, delimiter=";") rows = [row for row in reader] all_issues[project] = rows @@ -88,11 +88,16 @@ def merge_issues(all_issues): log.info(f"Total merged rows: {len(merged)}") return merged -def save_merged(merged_rows, output_path): +def save_merged(merged_rows, resdir, project_list, custom_dir): """ - Saves the file with the updated contents + Saves the merged file to a new directory alongside the input directory. """ + # Same directory as input directory with custom name - given by user + output_dir = os.path.join(os.path.abspath(resdir), custom_dir, "proximity") + os.makedirs(output_dir, exist_ok=True) + + output_path = os.path.join(output_dir, "issues-github.list") with open(output_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL) writer.writerows(merged_rows) - log.info(f"Saved to {output_path}") + log.info(f"Saved to {output_path}") From 7c1ac47781d9a83511da8dc5ca6e7d4bbcfa7a77 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Sun, 10 May 2026 13:31:09 +0200 Subject: [PATCH 66/79] file name and commit code updated --- .../__init__.py | 0 .../combine_projects.py | 74 ++++++++++++++----- ...mbine-issues.py => run-combine-projects.py | 4 +- 3 files changed, 59 insertions(+), 19 deletions(-) rename {combine_issues => combine_projects}/__init__.py (100%) rename combine_issues/combine_issues.py => combine_projects/combine_projects.py (60%) rename run-combine-issues.py => run-combine-projects.py (91%) diff --git a/combine_issues/__init__.py b/combine_projects/__init__.py similarity index 100% rename from combine_issues/__init__.py rename to combine_projects/__init__.py diff --git a/combine_issues/combine_issues.py b/combine_projects/combine_projects.py similarity index 60% rename from combine_issues/combine_issues.py rename to combine_projects/combine_projects.py index 7363a03..7b7c1aa 100644 --- a/combine_issues/combine_issues.py +++ b/combine_projects/combine_projects.py @@ -37,22 +37,24 @@ def run(): parser.add_argument( "--output", required= True, help="Custom output directory name" ) args = parser.parse_args() - # extract issues - all_issues = extract_issues(args.projects, args.resdir) - # merge and update the issue content - merged = merge_issues(all_issues) - # save merged issues - save_merged(merged, args.resdir, args.projects, args.output) - log.info("Issues successfully merged!") + files = ["commits.list","issues-github.list"] + for file in files: + # extract data + all_data = extract_data(args.projects, args.resdir, file) + # merge and update the issue content + merged_data = merge_data(all_data,file) + # save merged issues + save_merged(merged_data, args.resdir, args.output, file) + log.info(f"{file} data successfully merged!") -def extract_issues(project_list, threemonth_dir): +def extract_data(project_list, threemonth_dir,type_data): """ Extracts each issues-github.list data from each project and appends to all issues """ - all_issues = {} + all_data = {} for project in project_list: - # Matches the actual path for data: threemonth//proximity/issues-github.list - issues_file = os.path.join(threemonth_dir, project, "proximity", "issues-github.list") + # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) + issues_file = os.path.join(threemonth_dir, project, "proximity", type_data) if not os.path.exists(issues_file): log.warning(f"File not found: {issues_file}") continue @@ -60,9 +62,39 @@ def extract_issues(project_list, threemonth_dir): with open(issues_file, newline="", encoding="utf-8") as f: reader = csv.reader(f, delimiter=";") rows = [row for row in reader] - all_issues[project] = rows + all_data[project] = rows log.info(f"Loaded {len(rows)} rows from '{project}'") - return all_issues + return all_data + +def merge_data(all_data, file): + ''' + Merging data based on the file data + ''' + if file == "commits.list": + return merge_commits(all_data) + if file == "issues-github.list": + return merge_issues(all_data) + log.error("Incorrect file name!") + + +def merge_commits(all_commits): + ''' + All commit data is taken and updated to the required format + ''' + merged_commits = [] + for project, rows in all_commits.items(): + short_name = project.replace("_proximity","") + for row in rows: + if not row: + continue + new_row = row.copy() + new_row[0] = f"{short_name}-{new_row[0]}" + # update row 12 only if its not empty + if new_row[12] != "": + new_row[12] =f"{short_name}/{new_row[12]}" + merged_commits.append(new_row) + + return merged_commits def merge_issues(all_issues): """ @@ -76,19 +108,27 @@ def merge_issues(all_issues): continue new_row = row.copy() # Updating firts row: 1 -> keras-1 - new_row[0] = f"{short_name}_{new_row[0]}" + new_row[0] = f"{short_name}-{new_row[0]}" # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> keras-3885 last_col = new_row[13].strip().strip('"') + # checking if the 8th Column is "connected" + connected_col = new_row[8].strip().strip('"') + sub_issues = new_row[7].strip().strip('"') issue_num = new_row[12].strip().strip('"') - if last_col.lower() == "issue" and issue_num.isdigit(): + if (last_col.lower() == "issue" or connected_col.lower() == "connected" ) and issue_num.isdigit(): new_row[12] = f"{short_name}-{issue_num}" + + if sub_issues and sub_issues != '[]': + inner = sub_issues.strip('[]') + sub_issues_list = [s.strip() for s in inner.split(',')] + new_row[7] = str([f"{short_name}-{issue}" for issue in sub_issues_list]) merged.append(new_row) log.info(f"Total merged rows: {len(merged)}") return merged -def save_merged(merged_rows, resdir, project_list, custom_dir): +def save_merged(merged_rows, resdir, custom_dir, file): """ Saves the merged file to a new directory alongside the input directory. """ @@ -96,7 +136,7 @@ def save_merged(merged_rows, resdir, project_list, custom_dir): output_dir = os.path.join(os.path.abspath(resdir), custom_dir, "proximity") os.makedirs(output_dir, exist_ok=True) - output_path = os.path.join(output_dir, "issues-github.list") + output_path = os.path.join(output_dir, file) with open(output_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL) writer.writerows(merged_rows) diff --git a/run-combine-issues.py b/run-combine-projects.py similarity index 91% rename from run-combine-issues.py rename to run-combine-projects.py index 5518f7f..0646b57 100644 --- a/run-combine-issues.py +++ b/run-combine-projects.py @@ -14,6 +14,6 @@ # # Copyright 2026 by Ritika Hiremath # All Rights Reserved. -from combine_issues import combine_issues +from combine_projects import combine_projects -combine_issues.run() +combine_projects.run() From d5d0681036224e54abcdd9a57cd531de46b75725 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Mon, 25 May 2026 23:29:02 +0200 Subject: [PATCH 67/79] add output parameter into zulipdata extraction --- .../zulip_data_extraction.py | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py index b8e2114..d94b049 100644 --- a/zulip_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -37,13 +37,19 @@ # The location of zuliprc.txt file parser = argparse.ArgumentParser() parser.add_argument("--zulip-config", default=None, help="Path to zuliprc config file") -args = parser.parse_args(sys.argv[1:]) +parser.add_argument("--output", default=None, help="Path to the output directory") +args = parser.parse_args(sys.argv[1:]) -# Resolve path -if args.zulip_config: - config_path = args.zulip_config -else: - config_path = os.path.join(os.path.dirname(__file__), "zuliprc") +# Resolve path +if args.zulip_config: + config_path = args.zulip_config +else: + config_path = os.path.join(os.path.dirname(__file__), "zuliprc") + +if args.output: + output_path = os.path.join(args.output, "issues-zulip.json") +else: + output_path = os.path.join(os.path.dirname(__file__), "issues-zulip.json") # Raise error if file not found if not os.path.exists(config_path): @@ -52,6 +58,15 @@ client = zulip.Client(config_file= config_path) +def run(): + log.info("Starting Zulip data extraction") + # use exisiting zulip_streams_and_topics.josn file + if os.path.exists(os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json")): + streams_and_topics= os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + else: + streams_and_topics = topics_extraction() + messages_extraction_for_each_stream(streams_and_topics,output_path) + def safe_get_topics(stream_id): """ Fetches all the topics from Zulip rust. @@ -93,7 +108,8 @@ def topics_extraction(): time.sleep(0.5) - with open("zulip_streams_and_topics.json", "w", encoding="utf-8") as f: + topics_file = os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + with open(topics_file, "w", encoding="utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) log.info("Saved zulip_streams_and_topics.json") @@ -174,13 +190,7 @@ def messages_extraction_for_each_stream(streams_with_topics): }) # Save everything - output_file = "issues.json" - with open(output_file, "w") as f: + with open(output_path, "w", encoding="utf-8") as f: json.dump(final_output, f, indent=2) - log.info("\n Saved all stream messages to: '{}'".format(output_file)) - -def run(): - log.info("Starting Zulip data extraction") - streams_and_topics = topics_extraction() - messages_extraction_for_each_stream(streams_and_topics) + log.info("\n Saved all stream messages to: '{}'".format(output_path)) \ No newline at end of file From 98852ba5c6489347353f4a32b9ce5c358497f625 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Tue, 26 May 2026 10:46:20 +0200 Subject: [PATCH 68/79] update zulip data extraction --- .../zulip_data_extraction.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py index d94b049..ca93860 100644 --- a/zulip_data_extraction/zulip_data_extraction.py +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -47,9 +47,9 @@ config_path = os.path.join(os.path.dirname(__file__), "zuliprc") if args.output: - output_path = os.path.join(args.output, "issues-zulip.json") + output_path = os.path.join(args.output, "zulip.json") else: - output_path = os.path.join(os.path.dirname(__file__), "issues-zulip.json") + output_path = os.path.join(os.path.dirname(__file__), "zulip.json") # Raise error if file not found if not os.path.exists(config_path): @@ -60,12 +60,15 @@ def run(): log.info("Starting Zulip data extraction") - # use exisiting zulip_streams_and_topics.josn file + # use exisiting zulip_streams_and_topics.json file if os.path.exists(os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json")): - streams_and_topics= os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + streams_and_topics_path = os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + with open(streams_and_topics_path, "r", encoding="utf-8") as f: + streams_and_topics = json.load(f) + else: streams_and_topics = topics_extraction() - messages_extraction_for_each_stream(streams_and_topics,output_path) + messages_extraction_for_each_stream(streams_and_topics) def safe_get_topics(stream_id): """ @@ -76,7 +79,7 @@ def safe_get_topics(stream_id): """ while True: - resp = client.get_stream_topics(stream_id=stream_id) + resp = client.get_stream_topics(stream_id = stream_id) if resp["result"] == "success": return [t["name"] for t in resp["topics"]] elif resp["result"] == "error" and resp.get("code") == "RATE_LIMIT_HIT": @@ -109,7 +112,7 @@ def topics_extraction(): time.sleep(0.5) topics_file = os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") - with open(topics_file, "w", encoding="utf-8") as f: + with open(topics_file, "w", encoding = "utf-8") as f: json.dump(data, f, indent=2, ensure_ascii=False) log.info("Saved zulip_streams_and_topics.json") @@ -123,7 +126,7 @@ def load_stream_topics(file_path): :params: file_path : path of the file for streams and topics. :return: returns the content of the file. """ - with open(file_path, "r", encoding="utf-8") as f: + with open(file_path, "r", encoding = "utf-8") as f: return json.load(f) @@ -190,7 +193,7 @@ def messages_extraction_for_each_stream(streams_with_topics): }) # Save everything - with open(output_path, "w", encoding="utf-8") as f: - json.dump(final_output, f, indent=2) + with open(output_path, "w", encoding = "utf-8") as f: + json.dump(final_output, f, indent = 2) log.info("\n Saved all stream messages to: '{}'".format(output_path)) \ No newline at end of file From f9d0046e57191978823c5af7dfc35090a9e3014a Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Fri, 12 Jun 2026 15:27:21 +0200 Subject: [PATCH 69/79] update combine_projects with dealialized user data --- combine_projects/combine_projects.py | 301 ++++++++++++++++++++++++++- 1 file changed, 290 insertions(+), 11 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index 7b7c1aa..fd920ba 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -15,12 +15,15 @@ # Copyright 2026 by Ritika Hiremath # All Rights Reserved. """ -This file merges different issues_github.list files from different projects. +This file merges different commits.list files and issues_github.list files from different projects. """ # coding=utf-8 import os +import re import csv import argparse +import subprocess +import sys from pathlib import Path from logging import getLogger from codeface_utils.util import setup_logging @@ -35,26 +38,54 @@ def run(): parser.add_argument( "--resdir", required=True, help="Path to data/results/threemonth/" ) parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. vue_proximity keras_proximity" ) parser.add_argument( "--output", required= True, help="Custom output directory name" ) + parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthoirty") args = parser.parse_args() - - files = ["commits.list","issues-github.list"] + + files = ["commits.list","issues-github.list","bots.list"] for file in files: # extract data - all_data = extract_data(args.projects, args.resdir, file) + all_data = extract_data_per_project(args.projects, args.resdir, file) # merge and update the issue content merged_data = merge_data(all_data,file) + # if merged_data is None: + # continue # save merged issues save_merged(merged_data, args.resdir, args.output, file) log.info(f"{file} data successfully merged!") + # extracts all usernames.list and authors.list to a single user_data + user_data = extract_user_data(args.projects, args.resdir) + # save user_data to users.list in the output directory + output_dir = os.path.join(os.path.abspath(args.resdir), args.output, "proximity") + os.makedirs(output_dir, exist_ok=True) + users_list_path = os.path.join(output_dir, "users.list") + with open(users_list_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";") + writer.writerows(user_data) + log.info(f"Saved users.list to {users_list_path}") + + # save combined authors.list (id;name;email) for post-GitAuthority dedup + authors_data = extract_authors_for_list(args.projects, args.resdir) + authors_list_path = os.path.join(output_dir, "authors.list") + with open(authors_list_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";") + writer.writerows(authors_data) + log.info(f"Saved authors.list to {authors_list_path}") + + # run gitauthority and save the csv file + run_gitauthority(args.gitauthority, output_dir, args.output) + + # update saved files("commits.list","issues-github.list","bots.list", "authors.list") and resave them + update(output_dir, args.output) + -def extract_data(project_list, threemonth_dir,type_data): +def extract_data_per_project(project_list, dir,type_data): """ - Extracts each issues-github.list data from each project and appends to all issues + Extracts each (issues-github.list or commits.list) data from each project and appends to all issues """ all_data = {} for project in project_list: # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) - issues_file = os.path.join(threemonth_dir, project, "proximity", type_data) + issues_file = os.path.join(dir, project, "proximity", type_data) if not os.path.exists(issues_file): log.warning(f"File not found: {issues_file}") continue @@ -65,22 +96,88 @@ def extract_data(project_list, threemonth_dir,type_data): all_data[project] = rows log.info(f"Loaded {len(rows)} rows from '{project}'") return all_data + +def extract_user_data(project_list, dir): + """ + extracts data from authors.list and usernames.list + The data in all_data contains each row in the format [usernmae,name,email] + """ + all_data = [] + author_data_files = ["authors.list","usernames.list"] + for type_data in author_data_files: + for project in project_list: + # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) + user_file = os.path.join(dir, project, "proximity", type_data) + if not os.path.exists(user_file): + log.warning(f"File not found: {user_file}") + continue + + with open(user_file, newline="", encoding="utf-8") as f: + # [ "Hakim El Hattab", "hakim.elhattab@gmail.com", ""] for authors.list + reader = csv.reader(f, delimiter=";") + if type_data == "authors.list": + rows = [[row[1], row[2],""] for row in reader if row] + else: + rows = [[row[1], row[2], row[0]] for row in reader if row] + all_data.extend(rows) + log.info(f"Loaded {len(rows)} rows from '{project}'") + return all_data + + +def extract_authors_for_list(project_list, dir): + """ + Collect unique (name, email) pairs from all projects' authors.list files and + assign new sequential numeric IDs. Returns rows as [id, name, email]. + """ + seen = {} # (name, email) → assigned id + for project in project_list: + author_file = os.path.join(dir, project, "proximity", "authors.list") + if not os.path.exists(author_file): + log.warning(f"File not found: {author_file}") + continue + with open(author_file, newline="", encoding="utf-8") as f: + reader = csv.reader(f, delimiter=";") + for row in reader: + if row and len(row) >= 3: + key = (row[1].strip(), row[2].strip()) + if key not in seen: + seen[key] = len(seen) + 1 + return [[str(aid), name, email] for (name, email), aid in seen.items()] def merge_data(all_data, file): - ''' + """ Merging data based on the file data - ''' + """ if file == "commits.list": return merge_commits(all_data) if file == "issues-github.list": return merge_issues(all_data) + if file == "bots.list": + return merge_bots(all_data) log.error("Incorrect file name!") +def run_gitauthority(script: str, dir: str, project_name: str): + """ + Running the gitauthority script with all required files/data + """ + script_path = Path(os.path.join(script, "gitAuthority.py")) + input_file = os.path.join(dir, "users.list") + Path(dir).mkdir(parents=True, exist_ok=True) + clean_name = Path(project_name).stem + cmd = [sys.executable, str(script_path), + "--file", str(input_file), + "--name", clean_name, + "--output-dir", str(dir), + "--drop-boolean-column"] + print(f"[gitauthority] Running: {' '.join(cmd)}") + subprocess.run(cmd, check=True, cwd=str(script_path.parent)) + + def merge_commits(all_commits): - ''' + """ All commit data is taken and updated to the required format - ''' + """ merged_commits = [] for project, rows in all_commits.items(): short_name = project.replace("_proximity","") @@ -128,6 +225,188 @@ def merge_issues(all_issues): log.info(f"Total merged rows: {len(merged)}") return merged +def merge_bots(all_bots): + """ + Combines bots.list rows from all projects, deduplicating by entire row. + """ + seen = set() + merged = [] + for rows in all_bots.values(): + for row in rows: + if not row: + continue + key = tuple(row) + if key not in seen: + seen.add(key) + merged.append(row) + log.info(f"Total merged bot rows: {len(merged)}") + return merged + +def parse_name_email(value): + """ + Parse a gitAuthority identity string like: + 'Firstname Lastname ' + Returns (name, email) or (value, "") if format is unexpected. + """ + value = value.strip().strip('"') + match = re.match(r'^(.*?)\s*<([^>]+)>\s*$', value) + if match: + return match.group(1).strip(), match.group(2).strip() + return value, "" + + +def parse_gitauthority_csv(rows): + """ + Parse the gitAuthority CSV with format: + project ; original_author_id ; dealialized_author_id + + Returns: + identity_map : dict[(str, str), (str, str)] + (orig_name, orig_email) → (canon_name, canon_email) + Only contains entries where original and canonical differ. + """ + identity_map = {} + + for row in rows: + if len(row) < 3 or row[1].strip().strip('"') == "original_author_id": + continue # skip header or malformed rows + + original = row[1].strip().strip('"') + canonical = row[2].strip().strip('"') + + orig_name, orig_email = parse_name_email(original) + canon_name, canon_email = parse_name_email(canonical) + + if orig_name != canon_name or orig_email != canon_email: + identity_map[(orig_name, orig_email)] = (canon_name, canon_email) + + return identity_map + + +def update_issues_github(git_authority_csv, issues_github_rows): + """ + Update col 9 (name) and col 10 (email) in issues-github.list + using canonical identities from gitAuthority CSV. + """ + identity_map = parse_gitauthority_csv(git_authority_csv) + + updated_rows = [] + updated_count = 0 + + for row in issues_github_rows: + if not row or len(row) < 11: + updated_rows.append(row) + continue + + new_row = row.copy() + canon = identity_map.get(row[9].strip().strip('"'), row[10].strip().strip('"')) + if canon: + new_row[9] = canon[0] + new_row[10] = canon[1] + updated_count += 1 + + updated_rows.append(new_row) + + log.info(f"update_issues_github: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + + +def update_commits(git_authority_csv, commits_rows): + """ + Update the two set of user data (cols 2, 3), (cols 5, 6) in commits.list + using canonical identities from gitAuthority CSV. + """ + identity_map = parse_gitauthority_csv(git_authority_csv) + + updated_rows = [] + updated_count = 0 + + for row in commits_rows: + if not row or len(row) < 7: + updated_rows.append(row) + continue + + new_row = row.copy() + + canon = identity_map.get(row[2].strip().strip('"'), row[3].strip().strip('"')) + if canon: + new_row[2] = canon[0] + new_row[3] = canon[1] + updated_count += 1 + + canon = identity_map.get(row[5].strip().strip('"'), row[6].strip().strip('"')) + + if canon: + new_row[5] = canon[0] + new_row[6] = canon[1] + + updated_rows.append(new_row) + + log.info(f"update_commits: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + +def update_bots(git_authority_csv, bots_rows): + """ + Update the user data (cols 0, 1) in bots.list + using canonical identities from gitAuthority CSV. + """ + identity_map = parse_gitauthority_csv(git_authority_csv) + + updated_rows = [] + updated_count = 0 + + for row in bots_rows: + if not row or len(row) < 7: + updated_rows.append(row) + continue + + new_row = row.copy() + + canon = identity_map.get(row[0].strip().strip('"'), row[1].strip().strip('"')) + + if canon: + new_row[0] = canon[0] + new_row[1] = canon[1] + + updated_rows.append(new_row) + + log.info(f"update_commits: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + +def update(output_dir, project_name): + """ + Fetches the dealialized user data (merged_authors_{project_name}.csv). + Checks and updates each exisiting files with this dealialized user data. + """ + ga_filename = f"merged_authors_{project_name}.csv" + ga_path = os.path.join(os.path.abspath(output_dir), ga_filename) + if not os.path.exists(ga_path): + log.error(f"gitAuthority CSV not found: {ga_path}") + return + + with open(ga_path, newline="", encoding="utf-8") as f: + git_authority_csv = list(csv.reader(f, delimiter=";")) + + def update_file(path, updater, label): + """ + checks if the file exists then runs the command to update the files with dealialized user data. + """ + if os.path.exists(path): + with open(path, newline="", encoding="utf-8") as f: + rows = list(csv.reader(f, delimiter=";")) + updated = updater(git_authority_csv, rows) + with open(path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(updated) + log.info(f"{label} saved") + else: + log.warning(f"{label} not found in {output_dir}") + + update_file(os.path.join(output_dir, "issues-github.list"), update_issues_github, "issues-github.list") + update_file(os.path.join(output_dir, "commits.list"), update_commits, "commits.list") + update_file(os.path.join(output_dir, "bots.list"), update_bots, "bots.list") + + log.info("update complete!") + def save_merged(merged_rows, resdir, custom_dir, file): """ Saves the merged file to a new directory alongside the input directory. From 0b76135682df2e82c7e1f203037586d84668e82a Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 18 Jun 2026 12:44:29 +0200 Subject: [PATCH 70/79] combining authors after gitauthority fixed --- combine_projects/combine_projects.py | 210 +++++++++++++++------------ 1 file changed, 115 insertions(+), 95 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index fd920ba..f4a37fd 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -40,15 +40,19 @@ def run(): parser.add_argument( "--output", required= True, help="Custom output directory name" ) parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthoirty") args = parser.parse_args() - - files = ["commits.list","issues-github.list","bots.list"] + + files = ["commits.list","issues-github.list","bots.list","authors.list","issues-jira.list","issues-zulip.list","emails.list"] for file in files: # extract data all_data = extract_data_per_project(args.projects, args.resdir, file) + if not all_data: + # if file not present in the project, skip to the next one + log.warning(f"No project contained '{file}', skipping.") + continue # merge and update the issue content merged_data = merge_data(all_data,file) - # if merged_data is None: - # continue + if not merged_data: + continue # save merged issues save_merged(merged_data, args.resdir, args.output, file) log.info(f"{file} data successfully merged!") @@ -61,26 +65,18 @@ def run(): with open(users_list_path, "w", newline="", encoding="utf-8") as f: writer = csv.writer(f, delimiter=";") writer.writerows(user_data) - log.info(f"Saved users.list to {users_list_path}") - - # save combined authors.list (id;name;email) for post-GitAuthority dedup - authors_data = extract_authors_for_list(args.projects, args.resdir) - authors_list_path = os.path.join(output_dir, "authors.list") - with open(authors_list_path, "w", newline="", encoding="utf-8") as f: - writer = csv.writer(f, delimiter=";") - writer.writerows(authors_data) - log.info(f"Saved authors.list to {authors_list_path}") + log.info(f"Saved users data (users.list) to {users_list_path}") # run gitauthority and save the csv file run_gitauthority(args.gitauthority, output_dir, args.output) - # update saved files("commits.list","issues-github.list","bots.list", "authors.list") and resave them + # update saved files and resave them update(output_dir, args.output) def extract_data_per_project(project_list, dir,type_data): """ - Extracts each (issues-github.list or commits.list) data from each project and appends to all issues + Extracts each file's data from each project and appends to all issues """ all_data = {} for project in project_list: @@ -99,7 +95,7 @@ def extract_data_per_project(project_list, dir,type_data): def extract_user_data(project_list, dir): """ - extracts data from authors.list and usernames.list + Extracts data from authors.list and usernames.list The data in all_data contains each row in the format [usernmae,name,email] """ all_data = [] @@ -123,26 +119,6 @@ def extract_user_data(project_list, dir): log.info(f"Loaded {len(rows)} rows from '{project}'") return all_data - -def extract_authors_for_list(project_list, dir): - """ - Collect unique (name, email) pairs from all projects' authors.list files and - assign new sequential numeric IDs. Returns rows as [id, name, email]. - """ - seen = {} # (name, email) → assigned id - for project in project_list: - author_file = os.path.join(dir, project, "proximity", "authors.list") - if not os.path.exists(author_file): - log.warning(f"File not found: {author_file}") - continue - with open(author_file, newline="", encoding="utf-8") as f: - reader = csv.reader(f, delimiter=";") - for row in reader: - if row and len(row) >= 3: - key = (row[1].strip(), row[2].strip()) - if key not in seen: - seen[key] = len(seen) + 1 - return [[str(aid), name, email] for (name, email), aid in seen.items()] def merge_data(all_data, file): """ @@ -150,12 +126,24 @@ def merge_data(all_data, file): """ if file == "commits.list": return merge_commits(all_data) - if file == "issues-github.list": + if file == "issues-github.list" or file == "issues-jira.list" or file == "issues-zulip.list": return merge_issues(all_data) - if file == "bots.list": - return merge_bots(all_data) - log.error("Incorrect file name!") + # General case: just combine the rows line by line. + return merge_generic(all_data) +def merge_generic(all_data): + """ + Combines rows from all projects line by line, without any project-specific + transformation. Used for files that don't need special handling. + """ + merged = [] + for rows in all_data.values(): + for row in rows: + if not row: + continue + merged.append(row) + log.info(f"Total merged rows: {len(merged)}") + return merged def run_gitauthority(script: str, dir: str, project_name: str): """ @@ -225,23 +213,6 @@ def merge_issues(all_issues): log.info(f"Total merged rows: {len(merged)}") return merged -def merge_bots(all_bots): - """ - Combines bots.list rows from all projects, deduplicating by entire row. - """ - seen = set() - merged = [] - for rows in all_bots.values(): - for row in rows: - if not row: - continue - key = tuple(row) - if key not in seen: - seen.add(key) - merged.append(row) - log.info(f"Total merged bot rows: {len(merged)}") - return merged - def parse_name_email(value): """ Parse a gitAuthority identity string like: @@ -262,8 +233,8 @@ def parse_gitauthority_csv(rows): Returns: identity_map : dict[(str, str), (str, str)] - (orig_name, orig_email) → (canon_name, canon_email) - Only contains entries where original and canonical differ. + (orig_name, orig_email) → (dealialized_name, dealialized_email) + Only contains entries where original and dealialized differ. """ identity_map = {} @@ -272,23 +243,22 @@ def parse_gitauthority_csv(rows): continue # skip header or malformed rows original = row[1].strip().strip('"') - canonical = row[2].strip().strip('"') + dealialized = row[2].strip().strip('"') orig_name, orig_email = parse_name_email(original) - canon_name, canon_email = parse_name_email(canonical) + dealialized_name, dealialized_email = parse_name_email(dealialized) - if orig_name != canon_name or orig_email != canon_email: - identity_map[(orig_name, orig_email)] = (canon_name, canon_email) + if orig_name != dealialized_name or orig_email != dealialized_email: + identity_map[(orig_name, orig_email)] = (dealialized_name, dealialized_email) return identity_map -def update_issues_github(git_authority_csv, issues_github_rows): +def update_issues_github( issues_github_rows, identity_map): """ Update col 9 (name) and col 10 (email) in issues-github.list - using canonical identities from gitAuthority CSV. + using dealialized identities from gitAuthority CSV. """ - identity_map = parse_gitauthority_csv(git_authority_csv) updated_rows = [] updated_count = 0 @@ -299,10 +269,11 @@ def update_issues_github(git_authority_csv, issues_github_rows): continue new_row = row.copy() - canon = identity_map.get(row[9].strip().strip('"'), row[10].strip().strip('"')) - if canon: - new_row[9] = canon[0] - new_row[10] = canon[1] + # dealianlized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[9].strip().strip('"'), row[10].strip().strip('"'))) + if dealialized: + new_row[9] = dealialized[0] + new_row[10] = dealialized[1] updated_count += 1 updated_rows.append(new_row) @@ -311,12 +282,11 @@ def update_issues_github(git_authority_csv, issues_github_rows): return updated_rows -def update_commits(git_authority_csv, commits_rows): +def update_commits(commits_rows, identity_map): """ Update the two set of user data (cols 2, 3), (cols 5, 6) in commits.list - using canonical identities from gitAuthority CSV. + using dealialized identities from gitAuthority CSV. """ - identity_map = parse_gitauthority_csv(git_authority_csv) updated_rows = [] updated_count = 0 @@ -327,50 +297,88 @@ def update_commits(git_authority_csv, commits_rows): continue new_row = row.copy() - - canon = identity_map.get(row[2].strip().strip('"'), row[3].strip().strip('"')) - if canon: - new_row[2] = canon[0] - new_row[3] = canon[1] + # dealianlized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[2].strip().strip('"'), row[3].strip().strip('"'))) + if dealialized: + new_row[2] = dealialized[0] + new_row[3] = dealialized[1] updated_count += 1 - canon = identity_map.get(row[5].strip().strip('"'), row[6].strip().strip('"')) - - if canon: - new_row[5] = canon[0] - new_row[6] = canon[1] + # dealianlized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[5].strip().strip('"'), row[6].strip().strip('"'))) + if dealialized: + new_row[5] = dealialized[0] + new_row[6] = dealialized[1] updated_rows.append(new_row) log.info(f"update_commits: {updated_count}/{len(updated_rows)} rows updated") return updated_rows -def update_bots(git_authority_csv, bots_rows): +def update_bots(bots_rows, identity_map): """ Update the user data (cols 0, 1) in bots.list - using canonical identities from gitAuthority CSV. + using dealialized identities from gitAuthority CSV. """ - identity_map = parse_gitauthority_csv(git_authority_csv) updated_rows = [] updated_count = 0 for row in bots_rows: - if not row or len(row) < 7: + if not row or len(row) < 2: updated_rows.append(row) continue new_row = row.copy() + # dealianlized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[0].strip().strip('"'), row[1].strip().strip('"'))) + if dealialized: + new_row[0] = dealialized[0] + new_row[1] = dealialized[1] + updated_count += 1 + + updated_rows.append(new_row) + + log.info(f"update_bots: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + +def update_authors(authors_rows,identity_map): + """ + Update the user data in authors.list + using dealialized identities from gitAuthority CSV. + """ - canon = identity_map.get(row[0].strip().strip('"'), row[1].strip().strip('"')) - - if canon: - new_row[0] = canon[0] - new_row[1] = canon[1] + updated_rows = [] + updated_count = 0 + + for row in authors_rows: + if not row or len(row) < 3: + updated_rows.append(row) + continue + + new_row = row.copy() + # dealianlized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[1].strip().strip('"'), row[2].strip().strip('"'))) + if dealialized: + dealialized_name, dealialized_email = dealialized + # find the id of the dealialized data in authors_rows. + dealialized_row = next( + (r for r in authors_rows if len(r) >= 3 + and r[1].strip().strip('"') == dealialized_name + and r[2].strip().strip('"') == dealialized_email), + None + ) + # updating id of the dealized row. + if dealialized_row: + new_row[0] = dealialized_row[0] + # uncomment to update name and email. + # new_row[1] = dealialized_name + # new_row[2] = dealialized_email + updated_count += 1 updated_rows.append(new_row) - log.info(f"update_commits: {updated_count}/{len(updated_rows)} rows updated") + log.info(f"update_authors: {updated_count}/{len(updated_rows)} rows updated") return updated_rows def update(output_dir, project_name): @@ -386,15 +394,26 @@ def update(output_dir, project_name): with open(ga_path, newline="", encoding="utf-8") as f: git_authority_csv = list(csv.reader(f, delimiter=";")) + identity_map = parse_gitauthority_csv(git_authority_csv) + log.info(f"identity_map: {len(identity_map)} dealialized entries") + + # Save identity_map to a CSV for inspection + identity_map_path = os.path.join(output_dir, "identity_map_debug.csv") + with open(identity_map_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";") + writer.writerow(["orig_name", "orig_email", "dealialized_name", "dealialized_email"]) + for (orig_name, orig_email), (deal_name, deal_email) in identity_map.items(): + writer.writerow([orig_name, orig_email, deal_name, deal_email]) + log.info(f"identity_map saved to {identity_map_path}") def update_file(path, updater, label): """ - checks if the file exists then runs the command to update the files with dealialized user data. + Checks if the file exists then runs the command to update the files with dealialized user data. """ if os.path.exists(path): with open(path, newline="", encoding="utf-8") as f: rows = list(csv.reader(f, delimiter=";")) - updated = updater(git_authority_csv, rows) + updated = updater(rows, identity_map) with open(path, "w", newline="", encoding="utf-8") as f: csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(updated) log.info(f"{label} saved") @@ -404,6 +423,7 @@ def update_file(path, updater, label): update_file(os.path.join(output_dir, "issues-github.list"), update_issues_github, "issues-github.list") update_file(os.path.join(output_dir, "commits.list"), update_commits, "commits.list") update_file(os.path.join(output_dir, "bots.list"), update_bots, "bots.list") + update_file(os.path.join(output_dir, "authors.list"), update_authors, "authors.list") log.info("update complete!") From e07ba66ecc363a2073ca92b5cff36d59804a998f Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 1 Jul 2026 15:42:40 +0200 Subject: [PATCH 71/79] fix: use None check for user name in zulip_issue_processing --- issue_processing/zulip_issue_processing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py index 4ba6c1c..926d021 100644 --- a/issue_processing/zulip_issue_processing.py +++ b/issue_processing/zulip_issue_processing.py @@ -551,7 +551,7 @@ def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames= # ensure string representation for name and e-mail address username = str(user["username"]) - name = str(user["name"]) if "name" in user else username + name = str(user["name"]) if user["name"] is not None else username mail = str(user["email"]) # construct string for ID service and send query From 42ad47b9cd09acbebaa16d44c29106e617d66993 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Wed, 1 Jul 2026 16:08:01 +0200 Subject: [PATCH 72/79] addition of disambiguation-after-db.list and jira_issue_processing copyright resolved --- combine_projects/combine_projects.py | 37 +++++++++++++++++++---- issue_processing/jira_issue_processing.py | 3 -- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index f4a37fd..a2e1355 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -349,6 +349,7 @@ def update_authors(authors_rows,identity_map): """ updated_rows = [] + disambiguation_rows = [] updated_count = 0 for row in authors_rows: @@ -370,16 +371,25 @@ def update_authors(authors_rows,identity_map): ) # updating id of the dealized row. if dealialized_row: + old_id = row[0] + old_name = row[1] + old_email = row[2] + new_row[0] = dealialized_row[0] - # uncomment to update name and email. - # new_row[1] = dealialized_name - # new_row[2] = dealialized_email - updated_count += 1 + new_row[1] = dealialized_name + new_row[2] = dealialized_email + + if old_id != new_row[0] or old_name != new_row[1] or old_email != new_row[2]: + disambiguation_rows.append([ + new_row[0], new_row[1], new_row[2], + old_id, old_name, old_email + ]) + updated_count += 1 updated_rows.append(new_row) log.info(f"update_authors: {updated_count}/{len(updated_rows)} rows updated") - return updated_rows + return updated_rows,disambiguation_rows def update(output_dir, project_name): """ @@ -423,7 +433,22 @@ def update_file(path, updater, label): update_file(os.path.join(output_dir, "issues-github.list"), update_issues_github, "issues-github.list") update_file(os.path.join(output_dir, "commits.list"), update_commits, "commits.list") update_file(os.path.join(output_dir, "bots.list"), update_bots, "bots.list") - update_file(os.path.join(output_dir, "authors.list"), update_authors, "authors.list") + # handle authors.list separately to also write disambiguation file + authors_path = os.path.join(output_dir, "authors.list") + if os.path.exists(authors_path): + with open(authors_path, newline="", encoding="utf-8") as f: + rows = list(csv.reader(f, delimiter=";")) + updated_rows, disambiguation_rows = update_authors(rows, identity_map) + with open(authors_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(updated_rows) + log.info("authors.list saved") + if disambiguation_rows: + dis_path = os.path.join(output_dir, "disambiguation-after-db.list") + with open(dis_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(disambiguation_rows) + log.info("disambiguation-after-db.list saved") + else: + log.warning(f"authors.list not found in {output_dir}") log.info("update complete!") diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index ed101c4..9516ea3 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -19,11 +19,8 @@ # Copyright 2020-2021 by Thomas Bock # Copyright 2026 by Thomas Bock # Copyright 2023, 2025 by Maximilian Löffler -<<<<<<< HEAD # Copyright 2025-2026 by Leo Sendelbach -======= # Copyright 2025-2026 by Ritika Hiremath ->>>>>>> 32ac114 (updated copyright for logging import) # All Rights Reserved. """ This file is able to extract Jira issue data from xml files. From 5866196b71f86aa5dab57cb46f50cce974f9f2f8 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Sun, 5 Jul 2026 17:51:44 +0200 Subject: [PATCH 73/79] fix: typo fixes in combine projects script --- combine_projects/combine_projects.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index a2e1355..eea7e99 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -36,12 +36,12 @@ def run(): parser = argparse.ArgumentParser(description="Merge issues-github.list files") parser.add_argument( "--resdir", required=True, help="Path to data/results/threemonth/" ) - parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. vue_proximity keras_proximity" ) + parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. project1_proximity project2_proximity" ) parser.add_argument( "--output", required= True, help="Custom output directory name" ) - parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthoirty") + parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthority") args = parser.parse_args() - files = ["commits.list","issues-github.list","bots.list","authors.list","issues-jira.list","issues-zulip.list","emails.list"] + files = ["commits.list", "issues-github.list", "bots.list", "authors.list", "issues-jira.list", "issues-zulip.list", "emails.list"] for file in files: # extract data all_data = extract_data_per_project(args.projects, args.resdir, file) @@ -158,7 +158,7 @@ def run_gitauthority(script: str, dir: str, project_name: str): "--name", clean_name, "--output-dir", str(dir), "--drop-boolean-column"] - print(f"[gitauthority] Running: {' '.join(cmd)}") + log.info(f"[gitauthority] Running: {' '.join(cmd)}") subprocess.run(cmd, check=True, cwd=str(script_path.parent)) @@ -174,7 +174,7 @@ def merge_commits(all_commits): continue new_row = row.copy() new_row[0] = f"{short_name}-{new_row[0]}" - # update row 12 only if its not empty + # update column 12 only if its not empty if new_row[12] != "": new_row[12] =f"{short_name}/{new_row[12]}" merged_commits.append(new_row) @@ -192,10 +192,10 @@ def merge_issues(all_issues): if not row: continue new_row = row.copy() - # Updating firts row: 1 -> keras-1 + # Updating firts row: 1 -> project1-1 new_row[0] = f"{short_name}-{new_row[0]}" - # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> keras-3885 + # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> project1-3885 last_col = new_row[13].strip().strip('"') # checking if the 8th Column is "connected" connected_col = new_row[8].strip().strip('"') From b2fafa70972b1deea0392841e67fac077a8eafa4 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Tue, 7 Jul 2026 09:39:13 +0200 Subject: [PATCH 74/79] fix: remove duplicates update_authors --- combine_projects/combine_projects.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index eea7e99..1d15012 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -41,7 +41,7 @@ def run(): parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthority") args = parser.parse_args() - files = ["commits.list", "issues-github.list", "bots.list", "authors.list", "issues-jira.list", "issues-zulip.list", "emails.list"] + files = ["commits.list", "issues-github.list", "bots.list", "authors.list", "issues-jira.list", "issues-zulip.list", "emails.list", "usernames.list", "commitMessages.list"] for file in files: # extract data all_data = extract_data_per_project(args.projects, args.resdir, file) @@ -177,7 +177,7 @@ def merge_commits(all_commits): # update column 12 only if its not empty if new_row[12] != "": new_row[12] =f"{short_name}/{new_row[12]}" - merged_commits.append(new_row) + merged_commits.append(new_row) return merged_commits @@ -349,8 +349,9 @@ def update_authors(authors_rows,identity_map): """ updated_rows = [] - disambiguation_rows = [] + disambiguation_rows = [] updated_count = 0 + seen_ids = set() for row in authors_rows: if not row or len(row) < 3: @@ -362,7 +363,7 @@ def update_authors(authors_rows,identity_map): dealialized = identity_map.get((row[1].strip().strip('"'), row[2].strip().strip('"'))) if dealialized: dealialized_name, dealialized_email = dealialized - # find the id of the dealialized data in authors_rows. + # find the id of the dealialized data in authors_rows. dealialized_row = next( (r for r in authors_rows if len(r) >= 3 and r[1].strip().strip('"') == dealialized_name @@ -374,11 +375,11 @@ def update_authors(authors_rows,identity_map): old_id = row[0] old_name = row[1] old_email = row[2] - + new_row[0] = dealialized_row[0] new_row[1] = dealialized_name new_row[2] = dealialized_email - + if old_id != new_row[0] or old_name != new_row[1] or old_email != new_row[2]: disambiguation_rows.append([ new_row[0], new_row[1], new_row[2], @@ -386,6 +387,10 @@ def update_authors(authors_rows,identity_map): ]) updated_count += 1 + # multiple rows can dealialize to the same id; keep only the first. + if new_row[0] in seen_ids: + continue + seen_ids.add(new_row[0]) updated_rows.append(new_row) log.info(f"update_authors: {updated_count}/{len(updated_rows)} rows updated") From b35c5023cf5a578a32fe7ee5328fc372ef9ccc28 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Tue, 7 Jul 2026 10:18:26 +0200 Subject: [PATCH 75/79] update usernames.list after gitauthority --- combine_projects/combine_projects.py | 36 ++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index 1d15012..fd0f9a8 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -157,6 +157,7 @@ def run_gitauthority(script: str, dir: str, project_name: str): "--file", str(input_file), "--name", clean_name, "--output-dir", str(dir), + "--username", "--drop-boolean-column"] log.info(f"[gitauthority] Running: {' '.join(cmd)}") subprocess.run(cmd, check=True, cwd=str(script_path.parent)) @@ -254,6 +255,34 @@ def parse_gitauthority_csv(rows): return identity_map +def extract_usernames(rows): + """ + Build a deduplicated username;name;email list from the gitAuthority CSV. + Column layout (with --username --drop-boolean-column): + project ; original_author_id ; dealialized_author_id ; username + Only rows with a non-empty username are kept. + """ + seen = set() + usernames = [] + + for row in rows: + if len(row) < 4 or row[1].strip().strip('"') == "original_author_id": + continue # skip header or malformed rows + + username = row[3].strip().strip('"') + if not username: + continue + + name, email = parse_name_email(row[2].strip().strip('"')) + entry = (username, name, email) + if entry in seen: + continue + seen.add(entry) + usernames.append(list(entry)) + + return usernames + + def update_issues_github( issues_github_rows, identity_map): """ Update col 9 (name) and col 10 (email) in issues-github.list @@ -421,6 +450,13 @@ def update(output_dir, project_name): writer.writerow([orig_name, orig_email, deal_name, deal_email]) log.info(f"identity_map saved to {identity_map_path}") + # rebuild usernames.list from the gitAuthority output, deduplicated + usernames_rows = extract_usernames(git_authority_csv) + usernames_path = os.path.join(output_dir, "usernames.list") + with open(usernames_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(usernames_rows) + log.info(f"usernames.list saved with {len(usernames_rows)} unique rows") + def update_file(path, updater, label): """ Checks if the file exists then runs the command to update the files with dealialized user data. From c8e0d8459cf4517d02406eb7894e3e9f8f69899b Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Tue, 7 Jul 2026 11:29:40 +0200 Subject: [PATCH 76/79] fix: check for author presence in update_authors function --- combine_projects/combine_projects.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index fd0f9a8..7b78237 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -416,6 +416,10 @@ def update_authors(authors_rows,identity_map): ]) updated_count += 1 + # skip rows without an author name. + if not new_row[1] or not new_row[1].strip().strip('"'): + continue + # multiple rows can dealialize to the same id; keep only the first. if new_row[0] in seen_ids: continue From 8cb591aaf2bd93f96b0c7811222367a88b8d5005 Mon Sep 17 00:00:00 2001 From: RitikaHiremath Date: Thu, 16 Jul 2026 23:12:43 +0200 Subject: [PATCH 77/79] username.list and authors.list issue fix --- combine_projects/combine_projects.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index 7b78237..4d184e7 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -114,7 +114,7 @@ def extract_user_data(project_list, dir): if type_data == "authors.list": rows = [[row[1], row[2],""] for row in reader if row] else: - rows = [[row[1], row[2], row[0]] for row in reader if row] + rows = [[row[1], row[2], row[0]] for row in reader if row and not row[0] == "None"] all_data.extend(rows) log.info(f"Loaded {len(rows)} rows from '{project}'") return all_data @@ -270,7 +270,7 @@ def extract_usernames(rows): continue # skip header or malformed rows username = row[3].strip().strip('"') - if not username: + if not username or (username.lower() == "none"): continue name, email = parse_name_email(row[2].strip().strip('"')) @@ -283,7 +283,7 @@ def extract_usernames(rows): return usernames -def update_issues_github( issues_github_rows, identity_map): +def update_issues_github(issues_github_rows, identity_map): """ Update col 9 (name) and col 10 (email) in issues-github.list using dealialized identities from gitAuthority CSV. @@ -380,7 +380,7 @@ def update_authors(authors_rows,identity_map): updated_rows = [] disambiguation_rows = [] updated_count = 0 - seen_ids = set() + seen_identities = set() for row in authors_rows: if not row or len(row) < 3: @@ -420,10 +420,11 @@ def update_authors(authors_rows,identity_map): if not new_row[1] or not new_row[1].strip().strip('"'): continue - # multiple rows can dealialize to the same id; keep only the first. - if new_row[0] in seen_ids: + # the same user data can have different ids (e.g. merged from different projects), so dedupe by identity(name, email), not id. + identity = (new_row[1].strip().strip('"'), new_row[2].strip().strip('"')) + if identity in seen_identities: continue - seen_ids.add(new_row[0]) + seen_identities.add(identity) updated_rows.append(new_row) log.info(f"update_authors: {updated_count}/{len(updated_rows)} rows updated") From 14403ca1afb2aa9b3a19b4cbc0ef9ae90bf3928f Mon Sep 17 00:00:00 2001 From: Thomas Bock Date: Sat, 5 Sep 2026 20:02:41 +0200 Subject: [PATCH 78/79] Raise csv field size and fix variable names and log statements in combine_projects.py Signed-off-by: Thomas Bock --- combine_projects/combine_projects.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index 4d184e7..b48e493 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -32,6 +32,8 @@ setup_logging() log = getLogger(__name__) +# raise csv field size limit +csv.field_size_limit(sys.maxsize) def run(): parser = argparse.ArgumentParser(description="Merge issues-github.list files") @@ -76,21 +78,21 @@ def run(): def extract_data_per_project(project_list, dir,type_data): """ - Extracts each file's data from each project and appends to all issues + Extracts each file's data from each project and appends to all data """ all_data = {} for project in project_list: # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) - issues_file = os.path.join(dir, project, "proximity", type_data) - if not os.path.exists(issues_file): - log.warning(f"File not found: {issues_file}") + data_file = os.path.join(dir, project, "proximity", type_data) + if not os.path.exists(data_file): + log.warning(f"File not found: {data_file}") continue - with open(issues_file, newline="", encoding="utf-8") as f: + with open(data_file, newline="", encoding="utf-8") as f: reader = csv.reader(f, delimiter=";") rows = [row for row in reader] all_data[project] = rows - log.info(f"Loaded {len(rows)} rows from '{project}'") + log.info(f"Loaded {len(rows)} rows from {type_data} of '{project}'") return all_data def extract_user_data(project_list, dir): @@ -119,7 +121,7 @@ def extract_user_data(project_list, dir): log.info(f"Loaded {len(rows)} rows from '{project}'") return all_data - + def merge_data(all_data, file): """ Merging data based on the file data @@ -152,7 +154,7 @@ def run_gitauthority(script: str, dir: str, project_name: str): script_path = Path(os.path.join(script, "gitAuthority.py")) input_file = os.path.join(dir, "users.list") Path(dir).mkdir(parents=True, exist_ok=True) - clean_name = Path(project_name).stem + clean_name = Path(project_name).stem cmd = [sys.executable, str(script_path), "--file", str(input_file), "--name", clean_name, @@ -195,7 +197,7 @@ def merge_issues(all_issues): new_row = row.copy() # Updating firts row: 1 -> project1-1 new_row[0] = f"{short_name}-{new_row[0]}" - + # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> project1-3885 last_col = new_row[13].strip().strip('"') # checking if the 8th Column is "connected" @@ -205,7 +207,7 @@ def merge_issues(all_issues): if (last_col.lower() == "issue" or connected_col.lower() == "connected" ) and issue_num.isdigit(): new_row[12] = f"{short_name}-{issue_num}" - + if sub_issues and sub_issues != '[]': inner = sub_issues.strip('[]') sub_issues_list = [s.strip() for s in inner.split(',')] @@ -497,7 +499,7 @@ def update_file(path, updater, label): log.warning(f"authors.list not found in {output_dir}") log.info("update complete!") - + def save_merged(merged_rows, resdir, custom_dir, file): """ Saves the merged file to a new directory alongside the input directory. From 15a5dc37589510079ffc612269bef91365cc429d Mon Sep 17 00:00:00 2001 From: Thomas Bock Date: Sat, 12 Sep 2026 00:32:59 +0200 Subject: [PATCH 79/79] Remove duplicate entries in combined bots.list or authors.list, etc. Signed-off-by: Thomas Bock --- combine_projects/combine_projects.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py index b48e493..46949ec 100644 --- a/combine_projects/combine_projects.py +++ b/combine_projects/combine_projects.py @@ -13,6 +13,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright 2026 by Ritika Hiremath +# Copyright 2026 by Thomas Bock # All Rights Reserved. """ This file merges different commits.list files and issues_github.list files from different projects. @@ -24,6 +25,7 @@ import argparse import subprocess import sys +import json from pathlib import Path from logging import getLogger from codeface_utils.util import setup_logging @@ -139,10 +141,15 @@ def merge_generic(all_data): transformation. Used for files that don't need special handling. """ merged = [] + seen = set() for rows in all_data.values(): for row in rows: if not row: continue + key = json.dumps(row, sort_keys=True) + if key in seen: + continue + seen.add(key) merged.append(row) log.info(f"Total merged rows: {len(merged)}") return merged @@ -300,7 +307,7 @@ def update_issues_github(issues_github_rows, identity_map): continue new_row = row.copy() - # dealianlized: 0 -> name , 1 -> email + # dealialized: 0 -> name , 1 -> email dealialized = identity_map.get((row[9].strip().strip('"'), row[10].strip().strip('"'))) if dealialized: new_row[9] = dealialized[0] @@ -328,14 +335,14 @@ def update_commits(commits_rows, identity_map): continue new_row = row.copy() - # dealianlized: 0 -> name , 1 -> email + # dealialized: 0 -> name , 1 -> email dealialized = identity_map.get((row[2].strip().strip('"'), row[3].strip().strip('"'))) if dealialized: new_row[2] = dealialized[0] new_row[3] = dealialized[1] updated_count += 1 - # dealianlized: 0 -> name , 1 -> email + # dealialized: 0 -> name , 1 -> email dealialized = identity_map.get((row[5].strip().strip('"'), row[6].strip().strip('"'))) if dealialized: new_row[5] = dealialized[0] @@ -354,6 +361,7 @@ def update_bots(bots_rows, identity_map): updated_rows = [] updated_count = 0 + seen_rows = set() for row in bots_rows: if not row or len(row) < 2: @@ -361,13 +369,18 @@ def update_bots(bots_rows, identity_map): continue new_row = row.copy() - # dealianlized: 0 -> name , 1 -> email + # dealialized: 0 -> name , 1 -> email dealialized = identity_map.get((row[0].strip().strip('"'), row[1].strip().strip('"'))) if dealialized: new_row[0] = dealialized[0] new_row[1] = dealialized[1] updated_count += 1 + key = json.dumps(new_row, sort_keys=True) + if key in seen_rows: + continue + + seen_rows.add(key) updated_rows.append(new_row) log.info(f"update_bots: {updated_count}/{len(updated_rows)} rows updated") @@ -390,7 +403,7 @@ def update_authors(authors_rows,identity_map): continue new_row = row.copy() - # dealianlized: 0 -> name , 1 -> email + # dealialized: 0 -> name , 1 -> email dealialized = identity_map.get((row[1].strip().strip('"'), row[2].strip().strip('"'))) if dealialized: dealialized_name, dealialized_email = dealialized