diff --git a/.gitignore b/.gitignore index db79ec1..792941e 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.Rhistory .idea/ *.pyc +.DS_Store diff --git a/anonymization/anonymization.py b/anonymization/anonymization.py index f4bfecb..291a575 100644 --- a/anonymization/anonymization.py +++ b/anonymization/anonymization.py @@ -34,6 +34,7 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer # create logger diff --git a/author_postprocessing/author_postprocessing.py b/author_postprocessing/author_postprocessing.py index 2b54ef7..5840e4e 100644 --- a/author_postprocessing/author_postprocessing.py +++ b/author_postprocessing/author_postprocessing.py @@ -14,6 +14,7 @@ # # Copyright 2015-2017 by Claus Hunsen # Copyright 2020-2022 by Thomas Bock +# Copyright 2025-2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -47,12 +48,20 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer +from github_user_utils.github_user_utils import known_copilot_users, copilot_unified_name, copilot_unified_email, \ + is_github_noreply_author, github_user, github_email, \ + commit_added_event, mentioned_event, subscribed_event, \ + assigned_event, unassigned_event, review_requested_event, \ + review_request_removed_event, generate_botname_variants, quot_m + # create logger setup_logging() log = getLogger(__name__) +known_copilot_users_extended = generate_botname_variants(known_copilot_users) ## # RUN POSTPROCESSING ## @@ -81,7 +90,7 @@ def perform_data_backup(results_path, results_path_backup): copy(current_file, backup_file) -def fix_github_browser_commits(data_path, issues_github_list, commits_list, authors_list, emails_list, bots_list): +def fix_github_browser_commits(data_path, issues_github_list, commits_list, authors_list, emails_list, bots_list, unify_copilot_users=True): """ Replace the author "GitHub " in both commit and GitHub issue data by the correct author. The author "GitHub " is automatically inserted as the committer of a commit that is made when @@ -92,7 +101,7 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth "GitHub " are removed. Also "mentioned" or "subscribed" events in the GitHub issue data which reference the author "GitHub " are removed from the GitHub issue data. In addition, remove the author "GitHub " also from the author data and bot data and remove e-mails that have been sent - by this author. + by this author. This method also unifies all known copilot users into a single user if desired. :param data_path: the path to the project data that is to be fixed :param issues_github_list: file name of the github issue data @@ -100,26 +109,8 @@ def fix_github_browser_commits(data_path, issues_github_list, commits_list, auth :param authors_list: file name of the corresponding author data :param emails_list: file name of the corresponding email data :param bots_list: file name of the corresponding bot data + :param unify_copilot_users: whether to unify known copilot users into a single user """ - github_user = "GitHub" - github_email = "noreply@github.com" - commit_added_event = "commit_added" - mentioned_event = "mentioned" - subscribed_event = "subscribed" - - """ - Helper function to check whether a (name, e-mail) pair belongs to the author "GitHub ". - There are two options in Codeface how this can happen: - (1) Username is "GitHub" and e-mail address is "noreply@github.com" - (2) Username is "GitHub" and e-mail address has been replaced by Codeface, resulting in "GitHub.noreply@github.com" - - :param name: the name of the author to be checked - :param email: the email address of the author to be checked - :return: whether the given (name, email) pair belongs to the "GitHub " author - """ - def is_github_noreply_author(name, email): - return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) - # Check for all files in the result directory of the project whether they need to be adjusted for filepath, _, filenames in walk(data_path): @@ -128,20 +119,32 @@ def is_github_noreply_author(name, email): if authors_list in filenames: f = path.join(filepath, authors_list) log.info("Remove author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) author_data = csv_writer.read_from_csv(f) author_data_new = [] - + copilot_user_added = False for author in author_data: # keep author entry only if it should not be removed if not is_github_noreply_author(author[1], author[2]): - author_data_new.append(author) + # unify copilot author if desired + if unify_copilot_users and author[1] in known_copilot_users_extended: + if not copilot_user_added: + author[1] = copilot_unified_name + author[2] = copilot_unified_email + copilot_user_added = True + author_data_new.append(author) + else: + author_data_new.append(author) csv_writer.write_to_csv(f, author_data_new) # (2) Remove e-mails from author 'GitHub ' from all emails.list files if emails_list in filenames: f = path.join(filepath, emails_list) log.info("Remove emails from author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) email_data = csv_writer.read_from_csv(f) email_data_new = [] @@ -149,6 +152,10 @@ def is_github_noreply_author(name, email): for email in email_data: # keep author entry only if it should not be removed if not is_github_noreply_author(email[0], email[1]): + # unify copilot users if desired + if unify_copilot_users and email[0] in known_copilot_users_extended: + email[0] = copilot_unified_name + email[1] = copilot_unified_email email_data_new.append(email) else: log.warning("Remove email %s as it was sent by %s <%s>.", email[2], email[0], email[1]) @@ -159,6 +166,8 @@ def is_github_noreply_author(name, email): if commits_list in filenames: f = path.join(filepath, commits_list) log.info("Replace author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) commit_data = csv_writer.read_from_csv(f) for commit in commit_data: @@ -167,6 +176,13 @@ def is_github_noreply_author(name, email): if is_github_noreply_author(commit[5], commit[6]): commit[5] = commit[2] commit[6] = commit[3] + # unify copilot author if desired + if unify_copilot_users and commit[5] in known_copilot_users_extended: + commit[5] = copilot_unified_name + commit[6] = copilot_unified_email + if unify_copilot_users and commit[2] in known_copilot_users_extended: + commit[2] = copilot_unified_name + commit[3] = copilot_unified_email csv_writer.write_to_csv(f, commit_data) @@ -175,26 +191,45 @@ def is_github_noreply_author(name, email): if issues_github_list in filenames: f = path.join(filepath, issues_github_list) log.info("Replace author %s <%s> in %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) issue_data = csv_writer.read_from_csv(f) # read commit data commit_data_file = path.join(data_path, commits_list) commit_data = csv_writer.read_from_csv(commit_data_file) commit_hash_to_author = {commit[7]: commit[2:4] for commit in commit_data} - + author_name_to_data = {author[1]: author[1:3] for author in author_data_new} issue_data_new = [] - for event in issue_data: + # unify events to use a single copilot user for all events triggered by a known copilot user + if unify_copilot_users and event[9] in known_copilot_users_extended: + event[9] = copilot_unified_name + event[10] = copilot_unified_email + if unify_copilot_users and event[8] == commit_added_event and event[13][1:-1] in known_copilot_users_extended: + # for commit added events, also unify the referenced author in event info 2 if it is a known copilot user + event[13] = quot_m + copilot_unified_name + quot_m + elif unify_copilot_users and event[8] in (mentioned_event, subscribed_event, assigned_event, unassigned_event, + review_requested_event, review_request_removed_event) \ + and event[12] in known_copilot_users_extended: + # for mentioned/subscribed events, also unify the referenced user in event info 1 and 2 if it is a known copilot user + event[12] = copilot_unified_name + event[13] = quot_m + copilot_unified_email + quot_m # replace author if necessary if is_github_noreply_author(event[9], event[10]) and event[8] == commit_added_event: # extract commit hash from event info 1 commit_hash = event[12] - + # extract author name from event info 2 while cutting excess '"' + name = event[13][1:-1] # extract commit author from commit data, if available if commit_hash in commit_hash_to_author: event[9] = commit_hash_to_author[commit_hash][0] event[10] = commit_hash_to_author[commit_hash][1] issue_data_new.append(event) + elif name in author_name_to_data: + event[9] = author_name_to_data[name][0] + event[10] = author_name_to_data[name][1] + issue_data_new.append(event) else: # the added commit is not part of the commit data. In most cases, this is due to merge commits # appearing in another pull request, as Codeface does not keep track of merge commits. As we @@ -223,6 +258,9 @@ def is_github_noreply_author(name, email): if bots_list in filenames: f = path.join(filepath, bots_list) log.info("Remove author %s <%s> from %s ...", github_user, github_email, f) + if unify_copilot_users: + log.info("Also unify copilot users to %s <%s> in %s ...", copilot_unified_name, copilot_unified_email, f) + copilot_user_added = False bot_data = csv_writer.read_from_csv(f) bot_data_new = [] @@ -230,7 +268,15 @@ def is_github_noreply_author(name, email): for entry in bot_data: # keep bot entry only if it should not be removed if not is_github_noreply_author(entry[0], entry[1]): - bot_data_new.append(entry) + # unify copilot users if desired + if unify_copilot_users and entry[0] in known_copilot_users_extended: + if not copilot_user_added: + entry[0] = copilot_unified_name + entry[1] = copilot_unified_email + copilot_user_added = True + bot_data_new.append(entry) + else: + bot_data_new.append(entry) else: log.warning("Remove entry %s <%s> from bots list.", entry[0], entry[1]) @@ -267,9 +313,6 @@ def run_postprocessing(conf, resdir, backup_data): bugs_jira_list = "bugs-jira.list" bots_list = "bots.list" - # When looking at elements originating from json lists, we need to consider quotation marks around the string - quot_m = "\"" - data_path = path.join(resdir, conf["project"], conf["tagging"]) # Correctly replace author 'GitHub ' in the commit data and in "commit_added" events of the @@ -359,6 +402,9 @@ def run_postprocessing(conf, resdir, backup_data): if person[4] == issue_event[12] and (quot_m + person[5] + quot_m) == issue_event[13]: issue_event[12] = person[1] issue_event[13] = quot_m + person[2] + quot_m + # replace name in event info 2 if necessary + if quot_m + person[4] + quot_m == issue_event[13]: + issue_event[13] = quot_m + person[1] + quot_m csv_writer.write_to_csv(f, issue_data) @@ -425,8 +471,12 @@ def run_postprocessing(conf, resdir, backup_data): # the bot is already in the list, check if there are different predictions stored_bot = bot_names_and_emails[(bot[0], bot[1])] if stored_bot[2] != bot[2]: + # if either of the predictions is agent, keep agent + if (stored_bot[2] == "Agent" or bot[2] == "Agent"): + stored_bot[2] = "Agent" + bot_names_and_emails[(bot[0], bot[1])] = stored_bot # if either of the predictions is bot, keep bot - if (stored_bot[2] == "Bot" or bot[2] == "Bot"): + elif (stored_bot[2] == "Bot" or bot[2] == "Bot"): stored_bot[2] = "Bot" bot_names_and_emails[(bot[0], bot[1])] = stored_bot # otherwise, if either of the predictions is human, keep human diff --git a/bot_processing/bot_processing.py b/bot_processing/bot_processing.py index 9b18dd4..c51ed14 100644 --- a/bot_processing/bot_processing.py +++ b/bot_processing/bot_processing.py @@ -13,6 +13,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. # # Copyright 2021-2022 by Thomas Bock +# Copyright 2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler # All Rights Reserved. @@ -26,7 +27,9 @@ from logging import getLogger from codeface_utils.configuration import Configuration +from codeface_utils.util import setup_logging from csv_writer import csv_writer +from github_user_utils.github_user_utils import known_copilot_users, generate_botname_variants # create logger setup_logging() @@ -54,6 +57,7 @@ def run(): # (the known bots file is the file in which known bots have been added manually and project independent) __confdir = os.path.join(args.resdir, os.path.dirname(args.config)) __known_bots_file = os.path.abspath(os.path.join(__confdir, "known_github_bots.list")) + __known_agents_file = os.path.abspath(os.path.join(__confdir, "known_github_agents.list")) # run processing of bot data: # 1) load bot data @@ -61,7 +65,7 @@ def run(): # 2) load user data users = load_user_data(os.path.join(__resdir, "usernames.list")) # 3) update bot data with user data and additionally add known bots if they occur in the project - bots = add_user_data(bots, users, __known_bots_file) + bots = add_user_data(bots, users, __known_bots_file, __known_agents_file) # 4) dump result to disk print_to_disk(bots, __resdir) @@ -81,7 +85,7 @@ def load_bot_data(bot_file, header = True): # check if file exists and exit early if not if not os.path.exists(bot_file): - log.error("Bot file '{}' does not exist! Exiting early...".format(bot_file)) + log.error("Bot/Agent file '{}' does not exist (can be empty)! Exiting early...".format(bot_file)) sys.exit(-1) bot_data = csv_writer.read_from_csv(bot_file, delimiter=',') @@ -113,12 +117,13 @@ def load_user_data(user_data_file): return user_data -def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_reduced): +def check_with_known_bot_or_agent_list(known_bots_file, known_agents_file, bot_data, user_data, bot_data_reduced): """ Check whether there are known bots occurring in the project. If so, add them to the bots list or update the bots list accordingly. :param known_bots_file: the file path to the list of known bot data + :param known_agents_file: the file path to the list of known agent data :param bot_data: the bot data originating from the bot prediction :param user_data: a dictionary from the issue data which maps GitHub usernames to authors :param bot_data_reduced: the bot data after mapping GitHub user names to authors @@ -128,6 +133,7 @@ def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_red # Read the list of known bots known_bots = load_bot_data(known_bots_file, header = False) + known_agents = load_bot_data(known_agents_file, header = False) # Get the GitHub usernames of the bots predicted to be a bot predicted_bots = [bot[0] if len(bot) > 0 else "" for bot in bot_data] @@ -135,30 +141,62 @@ def check_with_known_bot_list(known_bots_file, bot_data, user_data, bot_data_red for bot in known_bots: # (1) check if a known bot occurs in the GitHub issue data but has not been predicted - if bot[0] not in predicted_bots and bot[0] in user_data: + bot_variation_predicted_bots = containing_bot_variation(bot[0], predicted_bots) + bot_variation_user_data = containing_bot_variation(bot[0], user_data) + if bot_variation_predicted_bots is None and bot_variation_user_data is not None: # add the known bot as a bot to the bots list additional_bot = dict() - additional_bot["user"] = user_data[bot[0]] + additional_bot["user"] = user_data[bot_variation_user_data] additional_bot["prediction"] = "Bot" bot_data_reduced.append(additional_bot) log.info("Add known bot '{}' to bot data.".format(additional_bot["user"])) # (2) handle known bots that are already present in the bots list - elif bot[0] in predicted_bots and bot[0] in user_data: + elif bot_variation_predicted_bots is not None and bot_variation_user_data is not None: # make sure that this bot has also been predicited to be bot for predicted_bot in bot_data_reduced: - if predicted_bot["user"] == user_data[bot[0]]: + if predicted_bot["user"] == user_data[bot_variation_user_data]: predicted_bot["prediction"] = "Bot" - log.info("Mark user '{}' as bot in the bot data.".format(user_data[bot[0]])) + log.info("Mark user '{}' as bot in the bot data.".format(user_data[bot_variation_user_data])) + break + + # get list of known agents and combine it with the list of known copilot users + copilot_users_variants = generate_botname_variants(known_copilot_users) + # get list of known agent names + known_agents_names = [agent[0] for agent in known_agents] + for copilot_user in copilot_users_variants: + if copilot_user not in known_agents_names: + known_agents.append([copilot_user]) + + for agent in known_agents: + + # (1) check if a known agent occurs in the GitHub issue data but has not been predicted + if agent[0] not in predicted_bots and agent[0] in user_data: + + # add the known agent as a bot to the bots list + additional_agent = dict() + additional_agent["user"] = user_data[agent[0]] + additional_agent["prediction"] = "Agent" + bot_data_reduced.append(additional_agent) + log.info("Add known agent '{}' to bot data.".format(additional_agent["user"])) + + # (2) handle known agents that are already present in the bots list + elif agent[0] in predicted_bots and agent[0] in user_data: + + # make sure that this bot has also been predicited to be an agent + for predicted_bot in bot_data_reduced: + if predicted_bot["user"] == user_data[agent[0]]: + predicted_bot["prediction"] = "Agent" + log.info("Mark user '{}' as agent in the bot data.".format(user_data[agent[0]])) break # return the updated bot data return bot_data_reduced -def add_user_data(bot_data, user_data, known_bots_file): +def add_user_data(bot_data, user_data, known_bots_file, known_agents_file): """ Add user data to bot data, i.e., replace username by name and e-mail. In addition, check in the global bots list whether there are authors in the projects which are @@ -194,19 +232,41 @@ def add_user_data(bot_data, user_data, known_bots_file): continue # get user information if available - if user[0] in list(user_buffer.keys()): - bot_reduced["user"] = user_buffer[user[0]] + bot_variation = containing_bot_variation(user[0], user_buffer.keys()) + if bot_variation is not None: + bot_reduced["user"] = user_buffer[bot_variation] bot_reduced["prediction"] = user[-1] bot_data_reduced.append(bot_reduced) else: log.warning("User '{}' in bot data does not occur in GitHub user data. Remove user...".format(user[0])) # check whether known GitHub bots occur in the GitHub issue data and, if so, update the bot data accordingly - bot_data_reduced = check_with_known_bot_list(known_bots_file, bot_data, user_buffer, bot_data_reduced) + bot_data_reduced = check_with_known_bot_or_agent_list(known_bots_file, known_agents_file, bot_data, user_buffer, bot_data_reduced) return bot_data_reduced +def containing_bot_variation(botname, name_list): + """ + Helper function to return the variation of a given bot name that occurs in a list of names. + + :param botname: the bot name for which the variation should be returned + :param name_list: the list of names to be checked for containing the bot name or a variation of it + :return: the variation of the given bot name that occurs in the given list of names, or None if no such variation exists + """ + + if botname in name_list: + return botname + elif botname + "bot" in name_list: + return botname + "bot" + elif botname + "[bot]" in name_list: + return botname + "[bot]" + elif botname.replace("[", "").replace("]", "") in name_list: + return botname.replace("[", "").replace("]", "") + else: + return None + + def print_to_disk(bot_data, results_folder): """ Print bot data to file "bots.list" in result folder. diff --git a/codeface_extraction/extractions.py b/codeface_extraction/extractions.py index 9c636dd..0210b73 100644 --- a/codeface_extraction/extractions.py +++ b/codeface_extraction/extractions.py @@ -15,6 +15,7 @@ # Copyright 2015-2018 by Claus Hunsen # Copyright 2016, 2018-2019 by Thomas Bock # Copyright 2019, 2021 by Thomas Bock +# Copyright 2026 by Thomas Bock # Copyright 2018 by Barbara Eckl # Copyright 2018 by Tina Schuh # Copyright 2025 by Maximilian Löffler @@ -759,18 +760,15 @@ def fix_name_encoding(name): if name is None: return name - # encode utf-8 - name = name.encode('utf-8') - # find out character set of the encoded name - info = decode_header(str(name)) + info = decode_header(name) try: # Apply correct encoding and return unicode string return str(make_header(info)) except UnicodeDecodeError: # Undo utf-8 encoding and return unicode string - return str(name.decode('utf-8')) + return name except LookupError: # Encoding not found, return string as is return name diff --git a/codeface_utils/cluster/idManager.py b/codeface_utils/cluster/idManager.py index 43a4be5..b7c3c0d 100644 --- a/codeface_utils/cluster/idManager.py +++ b/codeface_utils/cluster/idManager.py @@ -14,6 +14,7 @@ # Copyright 2010, 2011 by Wolfgang Mauerer # Copyright 2012, 2013 by Siemens AG, Wolfgang Mauerer # Copyright 2025 by Maximilian Löffler +# Copyright 2026 by Thomas Bock # All Rights Reserved. # # The code in this file originates from: @@ -225,7 +226,7 @@ def getPersonFromDB(self, person_id): log.exception("Could not reach ID service. Is the server running?\n") raise - result = res.read() + result = res.read().decode("utf-8") jsond = json.loads(result)[0] return (jsond) diff --git a/codeface_utils/configuration.py b/codeface_utils/configuration.py index e4a654a..49d35ab 100644 --- a/codeface_utils/configuration.py +++ b/codeface_utils/configuration.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/dbmanager.py b/codeface_utils/dbmanager.py index aecc172..407a931 100644 --- a/codeface_utils/dbmanager.py +++ b/codeface_utils/dbmanager.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +# coding=utf-8 # This file is part of Codeface. Codeface is free software: you can # redistribute it and/or modify it under the terms of the GNU General Public # License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/linktype.py b/codeface_utils/linktype.py index 617d11f..953ce4f 100644 --- a/codeface_utils/linktype.py +++ b/codeface_utils/linktype.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/codeface_utils/util.py b/codeface_utils/util.py index 59402d8..80c1b51 100644 --- a/codeface_utils/util.py +++ b/codeface_utils/util.py @@ -1,3 +1,4 @@ +# coding=utf-8 # This file is part of codeface-extraction, which is free software: you # can redistribute it and/or modify it under the terms of the GNU General # Public License as published by the Free Software Foundation, version 2. diff --git a/combine_projects/__init__.py b/combine_projects/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/combine_projects/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/combine_projects/combine_projects.py b/combine_projects/combine_projects.py new file mode 100644 index 0000000..46949ec --- /dev/null +++ b/combine_projects/combine_projects.py @@ -0,0 +1,528 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Ritika Hiremath +# Copyright 2026 by Thomas Bock +# All Rights Reserved. +""" +This file merges different commits.list files and issues_github.list files from different projects. +""" +# coding=utf-8 +import os +import re +import csv +import argparse +import subprocess +import sys +import json +from pathlib import Path +from logging import getLogger +from codeface_utils.util import setup_logging + +# create logger +setup_logging() +log = getLogger(__name__) + +# raise csv field size limit +csv.field_size_limit(sys.maxsize) + +def run(): + parser = argparse.ArgumentParser(description="Merge issues-github.list files") + parser.add_argument( "--resdir", required=True, help="Path to data/results/threemonth/" ) + parser.add_argument( "--projects", nargs="+", required=True, help="One or more project folder names, e.g. project1_proximity project2_proximity" ) + parser.add_argument( "--output", required= True, help="Custom output directory name" ) + parser.add_argument( "--gitauthority", required= True, help = "path to the cloned gitauthority") + args = parser.parse_args() + + files = ["commits.list", "issues-github.list", "bots.list", "authors.list", "issues-jira.list", "issues-zulip.list", "emails.list", "usernames.list", "commitMessages.list"] + for file in files: + # extract data + all_data = extract_data_per_project(args.projects, args.resdir, file) + if not all_data: + # if file not present in the project, skip to the next one + log.warning(f"No project contained '{file}', skipping.") + continue + # merge and update the issue content + merged_data = merge_data(all_data,file) + if not merged_data: + continue + # save merged issues + save_merged(merged_data, args.resdir, args.output, file) + log.info(f"{file} data successfully merged!") + # extracts all usernames.list and authors.list to a single user_data + user_data = extract_user_data(args.projects, args.resdir) + # save user_data to users.list in the output directory + output_dir = os.path.join(os.path.abspath(args.resdir), args.output, "proximity") + os.makedirs(output_dir, exist_ok=True) + users_list_path = os.path.join(output_dir, "users.list") + with open(users_list_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";") + writer.writerows(user_data) + log.info(f"Saved users data (users.list) to {users_list_path}") + + # run gitauthority and save the csv file + run_gitauthority(args.gitauthority, output_dir, args.output) + + # update saved files and resave them + update(output_dir, args.output) + + +def extract_data_per_project(project_list, dir,type_data): + """ + Extracts each file's data from each project and appends to all data + """ + all_data = {} + for project in project_list: + # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) + data_file = os.path.join(dir, project, "proximity", type_data) + if not os.path.exists(data_file): + log.warning(f"File not found: {data_file}") + continue + + with open(data_file, newline="", encoding="utf-8") as f: + reader = csv.reader(f, delimiter=";") + rows = [row for row in reader] + all_data[project] = rows + log.info(f"Loaded {len(rows)} rows from {type_data} of '{project}'") + return all_data + +def extract_user_data(project_list, dir): + """ + Extracts data from authors.list and usernames.list + The data in all_data contains each row in the format [usernmae,name,email] + """ + all_data = [] + author_data_files = ["authors.list","usernames.list"] + for type_data in author_data_files: + for project in project_list: + # Matches the actual path for data: threemonth//proximity/type_data(commits.list, issues-github.list) + user_file = os.path.join(dir, project, "proximity", type_data) + if not os.path.exists(user_file): + log.warning(f"File not found: {user_file}") + continue + + with open(user_file, newline="", encoding="utf-8") as f: + # [ "Hakim El Hattab", "hakim.elhattab@gmail.com", ""] for authors.list + reader = csv.reader(f, delimiter=";") + if type_data == "authors.list": + rows = [[row[1], row[2],""] for row in reader if row] + else: + rows = [[row[1], row[2], row[0]] for row in reader if row and not row[0] == "None"] + all_data.extend(rows) + log.info(f"Loaded {len(rows)} rows from '{project}'") + return all_data + + +def merge_data(all_data, file): + """ + Merging data based on the file data + """ + if file == "commits.list": + return merge_commits(all_data) + if file == "issues-github.list" or file == "issues-jira.list" or file == "issues-zulip.list": + return merge_issues(all_data) + # General case: just combine the rows line by line. + return merge_generic(all_data) + +def merge_generic(all_data): + """ + Combines rows from all projects line by line, without any project-specific + transformation. Used for files that don't need special handling. + """ + merged = [] + seen = set() + for rows in all_data.values(): + for row in rows: + if not row: + continue + key = json.dumps(row, sort_keys=True) + if key in seen: + continue + seen.add(key) + merged.append(row) + log.info(f"Total merged rows: {len(merged)}") + return merged + +def run_gitauthority(script: str, dir: str, project_name: str): + """ + Running the gitauthority script with all required files/data + """ + script_path = Path(os.path.join(script, "gitAuthority.py")) + input_file = os.path.join(dir, "users.list") + Path(dir).mkdir(parents=True, exist_ok=True) + clean_name = Path(project_name).stem + cmd = [sys.executable, str(script_path), + "--file", str(input_file), + "--name", clean_name, + "--output-dir", str(dir), + "--username", + "--drop-boolean-column"] + log.info(f"[gitauthority] Running: {' '.join(cmd)}") + subprocess.run(cmd, check=True, cwd=str(script_path.parent)) + + +def merge_commits(all_commits): + """ + All commit data is taken and updated to the required format + """ + merged_commits = [] + for project, rows in all_commits.items(): + short_name = project.replace("_proximity","") + for row in rows: + if not row: + continue + new_row = row.copy() + new_row[0] = f"{short_name}-{new_row[0]}" + # update column 12 only if its not empty + if new_row[12] != "": + new_row[12] =f"{short_name}/{new_row[12]}" + merged_commits.append(new_row) + + return merged_commits + +def merge_issues(all_issues): + """ + All issues are taken and corrected to the required format + """ + merged = [] + for project, rows in all_issues.items(): + short_name = project.replace("_proximity","") + for row in rows: + if not row: + continue + new_row = row.copy() + # Updating firts row: 1 -> project1-1 + new_row[0] = f"{short_name}-{new_row[0]}" + + # Checking last row is indeed """issue""" then updating the last but one row: 3885 -> project1-3885 + last_col = new_row[13].strip().strip('"') + # checking if the 8th Column is "connected" + connected_col = new_row[8].strip().strip('"') + sub_issues = new_row[7].strip().strip('"') + issue_num = new_row[12].strip().strip('"') + + if (last_col.lower() == "issue" or connected_col.lower() == "connected" ) and issue_num.isdigit(): + new_row[12] = f"{short_name}-{issue_num}" + + if sub_issues and sub_issues != '[]': + inner = sub_issues.strip('[]') + sub_issues_list = [s.strip() for s in inner.split(',')] + new_row[7] = str([f"{short_name}-{issue}" for issue in sub_issues_list]) + merged.append(new_row) + log.info(f"Total merged rows: {len(merged)}") + return merged + +def parse_name_email(value): + """ + Parse a gitAuthority identity string like: + 'Firstname Lastname ' + Returns (name, email) or (value, "") if format is unexpected. + """ + value = value.strip().strip('"') + match = re.match(r'^(.*?)\s*<([^>]+)>\s*$', value) + if match: + return match.group(1).strip(), match.group(2).strip() + return value, "" + + +def parse_gitauthority_csv(rows): + """ + Parse the gitAuthority CSV with format: + project ; original_author_id ; dealialized_author_id + + Returns: + identity_map : dict[(str, str), (str, str)] + (orig_name, orig_email) → (dealialized_name, dealialized_email) + Only contains entries where original and dealialized differ. + """ + identity_map = {} + + for row in rows: + if len(row) < 3 or row[1].strip().strip('"') == "original_author_id": + continue # skip header or malformed rows + + original = row[1].strip().strip('"') + dealialized = row[2].strip().strip('"') + + orig_name, orig_email = parse_name_email(original) + dealialized_name, dealialized_email = parse_name_email(dealialized) + + if orig_name != dealialized_name or orig_email != dealialized_email: + identity_map[(orig_name, orig_email)] = (dealialized_name, dealialized_email) + + return identity_map + + +def extract_usernames(rows): + """ + Build a deduplicated username;name;email list from the gitAuthority CSV. + Column layout (with --username --drop-boolean-column): + project ; original_author_id ; dealialized_author_id ; username + Only rows with a non-empty username are kept. + """ + seen = set() + usernames = [] + + for row in rows: + if len(row) < 4 or row[1].strip().strip('"') == "original_author_id": + continue # skip header or malformed rows + + username = row[3].strip().strip('"') + if not username or (username.lower() == "none"): + continue + + name, email = parse_name_email(row[2].strip().strip('"')) + entry = (username, name, email) + if entry in seen: + continue + seen.add(entry) + usernames.append(list(entry)) + + return usernames + + +def update_issues_github(issues_github_rows, identity_map): + """ + Update col 9 (name) and col 10 (email) in issues-github.list + using dealialized identities from gitAuthority CSV. + """ + + updated_rows = [] + updated_count = 0 + + for row in issues_github_rows: + if not row or len(row) < 11: + updated_rows.append(row) + continue + + new_row = row.copy() + # dealialized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[9].strip().strip('"'), row[10].strip().strip('"'))) + if dealialized: + new_row[9] = dealialized[0] + new_row[10] = dealialized[1] + updated_count += 1 + + updated_rows.append(new_row) + + log.info(f"update_issues_github: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + + +def update_commits(commits_rows, identity_map): + """ + Update the two set of user data (cols 2, 3), (cols 5, 6) in commits.list + using dealialized identities from gitAuthority CSV. + """ + + updated_rows = [] + updated_count = 0 + + for row in commits_rows: + if not row or len(row) < 7: + updated_rows.append(row) + continue + + new_row = row.copy() + # dealialized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[2].strip().strip('"'), row[3].strip().strip('"'))) + if dealialized: + new_row[2] = dealialized[0] + new_row[3] = dealialized[1] + updated_count += 1 + + # dealialized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[5].strip().strip('"'), row[6].strip().strip('"'))) + if dealialized: + new_row[5] = dealialized[0] + new_row[6] = dealialized[1] + + updated_rows.append(new_row) + + log.info(f"update_commits: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + +def update_bots(bots_rows, identity_map): + """ + Update the user data (cols 0, 1) in bots.list + using dealialized identities from gitAuthority CSV. + """ + + updated_rows = [] + updated_count = 0 + seen_rows = set() + + for row in bots_rows: + if not row or len(row) < 2: + updated_rows.append(row) + continue + + new_row = row.copy() + # dealialized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[0].strip().strip('"'), row[1].strip().strip('"'))) + if dealialized: + new_row[0] = dealialized[0] + new_row[1] = dealialized[1] + updated_count += 1 + + key = json.dumps(new_row, sort_keys=True) + if key in seen_rows: + continue + + seen_rows.add(key) + updated_rows.append(new_row) + + log.info(f"update_bots: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows + +def update_authors(authors_rows,identity_map): + """ + Update the user data in authors.list + using dealialized identities from gitAuthority CSV. + """ + + updated_rows = [] + disambiguation_rows = [] + updated_count = 0 + seen_identities = set() + + for row in authors_rows: + if not row or len(row) < 3: + updated_rows.append(row) + continue + + new_row = row.copy() + # dealialized: 0 -> name , 1 -> email + dealialized = identity_map.get((row[1].strip().strip('"'), row[2].strip().strip('"'))) + if dealialized: + dealialized_name, dealialized_email = dealialized + # find the id of the dealialized data in authors_rows. + dealialized_row = next( + (r for r in authors_rows if len(r) >= 3 + and r[1].strip().strip('"') == dealialized_name + and r[2].strip().strip('"') == dealialized_email), + None + ) + # updating id of the dealized row. + if dealialized_row: + old_id = row[0] + old_name = row[1] + old_email = row[2] + + new_row[0] = dealialized_row[0] + new_row[1] = dealialized_name + new_row[2] = dealialized_email + + if old_id != new_row[0] or old_name != new_row[1] or old_email != new_row[2]: + disambiguation_rows.append([ + new_row[0], new_row[1], new_row[2], + old_id, old_name, old_email + ]) + updated_count += 1 + + # skip rows without an author name. + if not new_row[1] or not new_row[1].strip().strip('"'): + continue + + # the same user data can have different ids (e.g. merged from different projects), so dedupe by identity(name, email), not id. + identity = (new_row[1].strip().strip('"'), new_row[2].strip().strip('"')) + if identity in seen_identities: + continue + seen_identities.add(identity) + updated_rows.append(new_row) + + log.info(f"update_authors: {updated_count}/{len(updated_rows)} rows updated") + return updated_rows,disambiguation_rows + +def update(output_dir, project_name): + """ + Fetches the dealialized user data (merged_authors_{project_name}.csv). + Checks and updates each exisiting files with this dealialized user data. + """ + ga_filename = f"merged_authors_{project_name}.csv" + ga_path = os.path.join(os.path.abspath(output_dir), ga_filename) + if not os.path.exists(ga_path): + log.error(f"gitAuthority CSV not found: {ga_path}") + return + + with open(ga_path, newline="", encoding="utf-8") as f: + git_authority_csv = list(csv.reader(f, delimiter=";")) + identity_map = parse_gitauthority_csv(git_authority_csv) + log.info(f"identity_map: {len(identity_map)} dealialized entries") + + # Save identity_map to a CSV for inspection + identity_map_path = os.path.join(output_dir, "identity_map_debug.csv") + with open(identity_map_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";") + writer.writerow(["orig_name", "orig_email", "dealialized_name", "dealialized_email"]) + for (orig_name, orig_email), (deal_name, deal_email) in identity_map.items(): + writer.writerow([orig_name, orig_email, deal_name, deal_email]) + log.info(f"identity_map saved to {identity_map_path}") + + # rebuild usernames.list from the gitAuthority output, deduplicated + usernames_rows = extract_usernames(git_authority_csv) + usernames_path = os.path.join(output_dir, "usernames.list") + with open(usernames_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(usernames_rows) + log.info(f"usernames.list saved with {len(usernames_rows)} unique rows") + + def update_file(path, updater, label): + """ + Checks if the file exists then runs the command to update the files with dealialized user data. + """ + if os.path.exists(path): + with open(path, newline="", encoding="utf-8") as f: + rows = list(csv.reader(f, delimiter=";")) + updated = updater(rows, identity_map) + with open(path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(updated) + log.info(f"{label} saved") + else: + log.warning(f"{label} not found in {output_dir}") + + update_file(os.path.join(output_dir, "issues-github.list"), update_issues_github, "issues-github.list") + update_file(os.path.join(output_dir, "commits.list"), update_commits, "commits.list") + update_file(os.path.join(output_dir, "bots.list"), update_bots, "bots.list") + # handle authors.list separately to also write disambiguation file + authors_path = os.path.join(output_dir, "authors.list") + if os.path.exists(authors_path): + with open(authors_path, newline="", encoding="utf-8") as f: + rows = list(csv.reader(f, delimiter=";")) + updated_rows, disambiguation_rows = update_authors(rows, identity_map) + with open(authors_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(updated_rows) + log.info("authors.list saved") + if disambiguation_rows: + dis_path = os.path.join(output_dir, "disambiguation-after-db.list") + with open(dis_path, "w", newline="", encoding="utf-8") as f: + csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL).writerows(disambiguation_rows) + log.info("disambiguation-after-db.list saved") + else: + log.warning(f"authors.list not found in {output_dir}") + + log.info("update complete!") + +def save_merged(merged_rows, resdir, custom_dir, file): + """ + Saves the merged file to a new directory alongside the input directory. + """ + # Same directory as input directory with custom name - given by user + output_dir = os.path.join(os.path.abspath(resdir), custom_dir, "proximity") + os.makedirs(output_dir, exist_ok=True) + + output_path = os.path.join(output_dir, file) + with open(output_path, "w", newline="", encoding="utf-8") as f: + writer = csv.writer(f, delimiter=";", quoting=csv.QUOTE_ALL) + writer.writerows(merged_rows) + log.info(f"Saved to {output_path}") diff --git a/github_user_utils/__init__.py b/github_user_utils/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/github_user_utils/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/github_user_utils/github_user_utils.py b/github_user_utils/github_user_utils.py new file mode 100644 index 0000000..3db2efd --- /dev/null +++ b/github_user_utils/github_user_utils.py @@ -0,0 +1,78 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Leo Sendelbach +# All Rights Reserved. +""" +This file serves as a collection of global variables and utility functions, which are used throughout the +issue data extraction and post-processing, in particular for the processing of GitHub and Copilot user data. +""" + +## +# GLOBAL VARIABLES +## + +# global variables containing all known copilot users and the name and mail address copilot users will be assigned +known_copilot_users = {"Copilot", "copilot-pull-request-reviewer[bot]", "copilot-swe-agent[bot]"} +copilot_unified_name = "Copilot" +copilot_unified_email = "copilot@example.com" + +## global variables for the GitHub author +github_user = "GitHub" +github_email = "noreply@github.com" +commit_added_event = "commit_added" +mentioned_event = "mentioned" +subscribed_event = "subscribed" +assigned_event = "assigned" +unassigned_event = "unassigned" +review_requested_event = "review_requested" +review_request_removed_event = "review_request_removed" + +# When looking at elements originating from json lists, we need to consider quotation marks around the string +quot_m = "\"" + +## +# UTILITY FUNCTIONS +## + +def is_github_noreply_author(name, email): + """ + Helper function to check whether a (name, e-mail) pair belongs to the author "GitHub ". + There are two options in Codeface how this can happen: + (1) Username is "GitHub" and e-mail address is "noreply@github.com" + (2) Username is "GitHub" and e-mail address has been replaced by Codeface, resulting in "GitHub.noreply@github.com" + + :param name: the name of the author to be checked + :param email: the email address of the author to be checked + :return: whether the given (name, email) pair belongs to the "GitHub " author + """ + + return (name == github_user and (email == github_email or email == (github_user + "." + github_email))) + +def generate_botname_variants(botnames): + """ + Helper function to generate variants of bot names, which are used in the list of + known bots and agents as well as during author postprocessing. + + :param botnames: the list of bot names for which variants should be generated + :return: a set of bot name variants + """ + + botname_variants = set() + for botname in botnames: + botname_variants.add(botname) + botname = botname.replace("[", "").replace("]", "") + botname_variants.add(botname) + + return botname_variants diff --git a/issue_processing/issue_processing.py b/issue_processing/issue_processing.py index 3db14d5..da0169a 100644 --- a/issue_processing/issue_processing.py +++ b/issue_processing/issue_processing.py @@ -18,8 +18,10 @@ # Copyright 2018-2019 by Anselm Fehnker # Copyright 2019 by Thomas Bock # Copyright 2020-2021 by Thomas Bock +# Copyright 2025-2026 by Leo Sendelbach # Copyright 2026 by Thomas Bock # Copyright 2025 by Maximilian Löffler +# Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. """ This file is able to extract Github issue data from json files. @@ -30,21 +32,24 @@ import os import sys from datetime import datetime, timedelta +import math from logging import getLogger from codeface_utils.cluster.idManager import dbIdManager, csvIdManager from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager +from codeface_utils.util import setup_logging from dateutil import parser as dateparser - +from codeface_utils.util import setup_logging from csv_writer import csv_writer +from github_user_utils.github_user_utils import copilot_unified_name # create logger setup_logging() log = getLogger(__name__) # known types from JIRA and GitHub default labels -known_types = {"bug", "improvement", "enhancement", "new feature", "task", "test", "wish"} +known_types = {"bug", "improvement", "enhancement", "feature", "task", "test", "wish"} # known resolutions from JIRA and GitHub default labels known_resolutions = {"unresolved", "fixed", "wontfix", "duplicate", "invalid", "incomplete", "cannot reproduce", @@ -55,6 +60,7 @@ # datetime format string datetime_format = "%Y-%m-%d %H:%M:%S" + def run(): # get all needed paths and arguments for the method call. parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') @@ -77,13 +83,14 @@ def run(): # 1) load the list of issues issues = load(__srcdir) # 2) re-format the issues - issues = reformat_issues(issues) + reformat_issues(issues) # 3) merges all issue events into one list - issues = merge_issue_events(issues) + external_connected_events = dict() + filtered_connected_events = merge_issue_events(issues, external_connected_events) # 4) re-format the eventsList of the issues - issues = reformat_events(issues) + reformat_events(issues, filtered_connected_events, external_connected_events) # 5) update user data with Codeface database and dump username-to-name/e-mail list - issues = insert_user_data(issues, __conf, __resdir) + insert_user_data(issues, __conf, __resdir) # 6) dump result to disk print_to_disk(issues, __resdir) @@ -232,7 +239,6 @@ def reformat_issues(issue_data): Re-arrange issue data structure. :param issue_data: the issue data to re-arrange - :return: the re-arranged issue data """ log.info("Re-arranging Github issues...") @@ -241,7 +247,10 @@ def reformat_issues(issue_data): for issue in issue_data: # empty container for issue types - issue["type"] = [] + if issue["type"] is None: + issue["type"] = [] + else: + issue["type"] = [issue["type"]["name"].lower()] # empty container for issue resolutions issue["resolution"] = [] @@ -258,14 +267,18 @@ def reformat_issues(issue_data): if issue["relatedCommits"] is None: issue["relatedCommits"] = [] - # if an issue has no reviewsList, an empty Listgets created + # if an issue has no reviewsList, an empty List gets created if issue["reviewsList"] is None: issue["reviewsList"] = [] # if an issue has no relatedIssues, an empty List gets created - if "relatedIssues" not in issue: + if issue["relatedIssues"] is None: issue["relatedIssues"] = [] + # if an issue has no sub-issue list, an empty List gets created + if issue["subIssues"] is None: + issue["subIssues"] = [] + # add "closed_at" information if not present yet if issue["closed_at"] is None: issue["closed_at"] = "" @@ -282,20 +295,22 @@ def reformat_issues(issue_data): else: issue["type"].append("issue") - return issue_data + return -def merge_issue_events(issue_data): +def merge_issue_events(issue_data, external_connected_events): """ All issue events are merged together in the eventsList. This simplifies processing in later steps. :param issue_data: the issue data from which the events shall be merged - :return: the issue data with merged eventsList + :param external_connected_events: a dict to store connected events to external issues + :return: a filtered dict of connected events for future reconstruction """ log.info("Merge issue events ...") issue_data_to_update = dict() + connected_events = dict() for issue in issue_data: @@ -364,6 +379,7 @@ def merge_issue_events(issue_data): # it is a commit which was added to the pull request if rel_commit["type"] == "commitAddedToPullRequest": rel_commit["event"] = "commit_added" + rel_commit["event_info_2"] = rel_commit["commit"]["author"] # if the related commit was mentioned in an issue comment: elif rel_commit["type"] == "commitMentionedInIssue": @@ -479,6 +495,12 @@ def merge_issue_events(issue_data): if event["event"] == "review_requested" or event["event"] == "review_request_removed": event["ref_target"] = event["requestedReviewer"] + # if event is a specific copilot event, assign the copilot user data + if event["event"] == "copilot_work_started" or event["event"] == "copilot_work_finished": + event["user"]["name"] = None + event["user"]["username"] = copilot_unified_name + event["user"]["email"] = "" + # if event dismisses a review, we can determine the original state of the corresponding review if event["event"] == "review_dismissed": for review in issue["reviewsList"]: @@ -491,6 +513,36 @@ def merge_issue_events(issue_data): event["ref_target"] = event["user"] event["user"] = event["assigner"] + # if event is merged event, save the hash of the merge commit in event_info_1 + if event["event"] == "merged" and not event["commit"] is None: + event["event_info_1"] = event["commit"]["hash"] + + # if event is connected event, create or add to a matching dict entry by matching timestamps, for later reconstruction + if event["event"] == "connected": + if event["created_at"] in list(connected_events.keys()) and connected_events[event["created_at"]]["user"] == event["user"]: + # if there is already a connected event at this time by this user, add this event to the list + connected_events[event["created_at"]]["issues"].append(issue["number"]) + elif subtract_seconds_from_time(event["created_at"], 1) in list(connected_events.keys()) \ + and connected_events[subtract_seconds_from_time(event["created_at"], 1)]["user"] == event["user"]: + # same as above, but accounting for a possible difference in timestamps of 1 second between matching events + connected_events[subtract_seconds_from_time(event["created_at"], 1)]["issues"].append(issue["number"]) + event["created_at"] = subtract_seconds_from_time(event["created_at"], 1) + elif subtract_seconds_from_time(event["created_at"], -1) in list(connected_events.keys()) \ + and connected_events[subtract_seconds_from_time(event["created_at"], -1)]["user"] == event["user"]: + # same as above, with offset calculated in the other direction + connected_events[subtract_seconds_from_time(event["created_at"], -1)]["issues"].append(issue["number"]) + event["created_at"] = subtract_seconds_from_time(event["created_at"], -1) + else: + # if there is no connected event yet at this timestamp, create a new entry for this event + connected_info = dict() + connected_info["issues"] = [issue["number"]] + connected_info["user"] = event["user"] + connected_events[event["created_at"]] = connected_info + + # if event is a locked event, save the lock reason in event_info_1 + if event["event"] == "locked": + event["event_info_1"] = event["lock_reason"] + # merge events, relatedCommits, relatedIssues and comment lists issue["eventsList"] = issue["commentsList"] + issue["eventsList"] + issue["relatedIssues"] + issue[ "relatedCommits"] + issue["reviewsList"] @@ -502,21 +554,62 @@ def merge_issue_events(issue_data): # sorts eventsList by time issue["eventsList"] = sorted(issue["eventsList"], key=lambda k: k["created_at"]) + # filter out connected events which cannot be perfectly matched + # and populate external_connected_events dict + # because this happens in place, we do not need to return the external_connected_event dict later + filtered_connected_events = dict(filter(lambda item: filter_connected_events(item[0], item[1], external_connected_events), connected_events.items())) + # updates all the issues by the temporarily stored referenced_by events for _, value in issue_data_to_update.items(): for issue in issue_data: if issue["number"] == value["number"]: issue["eventsList"] = issue["eventsList"] + value["eventsList"] - return issue_data - - -def reformat_events(issue_data): + # return the filtered_connected_events dict for later reconstruction + return filtered_connected_events + + +def filter_connected_events(key, value, external_connected_events): + num_issues = len(value["issues"]) + # if only a single connected event exists at this time, it has to be connecting to an external issue + if num_issues == 1: + external_connected_events[key] = value + return False + # if 2 connected events exist, matching them is trivial + if num_issues == 2: + return True + occurrences = {x: value["issues"].count(x) for x in set(value["issues"])} + # otherwise, if it is an even number, check if it can be easily matched, + # meaning that exactly half the events occur in the same issue + if num_issues % 2 == 0 and num_issues/2 in occurrences.values(): + # duplicate issue list for matching the issues later + value["multi_issues_copy"] = list(value["issues"]) + return True + # if it is an odd number, check if it can be easily matched + # meaning that exactly half (rounded up) the events occur in the same issue + if num_issues % 2 == 1 and (num_issues + 1)/2 in occurrences.values(): + for sub_key, sub_value in occurrences.items(): + # then, assign one of them as an external connected event and proceed as in previous case + if sub_value == (num_issues + 1)/2: + new_entry = dict() + new_entry["user"] = value["user"] + new_entry["issues"] = [sub_key] + external_connected_events[key] = new_entry + value["issues"].remove(sub_key) + # duplicate issue list for matching the issues later + value["multi_issues_copy"] = list(value["issues"]) + return True + # no other variants can be easily matched + return False + + +def reformat_events(issue_data, filtered_connected_events, external_connected_events): """ Re-format event information dependent on the event type. :param issue_data: the data of all issues that shall be re-formatted - :return: the issue data with updated event information + :param filtered_connected_events: the dict of connected events which can be reconstructed + :param external_connected_events: the dict of connected events to external issues """ log.info("Update event information ...") @@ -541,6 +634,35 @@ def reformat_events(issue_data): if event["ref_target"] is not None and not event["ref_target"] == "": users = update_user_dict(users, event["ref_target"]) + # reconstruction of connections + if event["event"] == "connected": + if event["created_at"] in external_connected_events \ + and issue["number"] in external_connected_events[event["created_at"]]["issues"]: + # if the event is an external connected event, mark it as such and remove this issue from the list + event["event_info_1"] = "external" + external_connected_events[event["created_at"]]["issues"].remove(issue["number"]) + elif event["created_at"] in filtered_connected_events \ + and issue["number"] in filtered_connected_events[event["created_at"]]["issues"]: + # if it is instead an internal connected event + value = filtered_connected_events[event["created_at"]] + if len(value["issues"]) == 2: + # and we only have 2 issues in the list, connect to the other issue + event["event_info_1"] = value["issues"][0] if value["issues"][1] == issue["number"] else value["issues"][1] + else: + # and we have more than two issues, count each issue's occurrences + occurrences = {x: value["issues"].count(x) for x in set(value["issues"])} + if occurrences[issue["number"]] == max(occurrences.values()): + # if our issue is the most common one, that means it is the common denominator + # for all connected events at this time + # so this event connects to any other issue + # which is then removed from a copied list to avoid duplications + number = next(x for x in value["multi_issues_copy"] if x != issue["number"]) + value["multi_issues_copy"].remove(number) + event["event_info_1"] = number + else: + # otherwise, connect this event to the common denominator + event["event_info_1"] = max(occurrences, key=occurrences.get) + # as the user dictionary is created, start re-formating the event information of all issues for issue in issue_data: @@ -558,13 +680,16 @@ def reformat_events(issue_data): if event["event"] == "closed": event["event"] = "state_updated" event["event_info_1"] = "closed" # new state - event["event_info_2"] = "open" # old state + if event["commit"] is not None: + event["event_info_2"] = event["commit"]["hash"] + else: + event["event_info_2"] = event["state_reason"] issue["state_new"] = "closed" elif event["event"] == "reopened": event["event"] = "state_updated" event["event_info_1"] = "open" # new state - event["event_info_2"] = "closed" # old state + event["event_info_2"] = event["state_reason"] issue["state_new"] = "reopened" elif event["event"] == "labeled": @@ -572,7 +697,7 @@ def reformat_events(issue_data): event["event_info_1"] = label # if the label is in this list, it also is a type of the issue - if label in known_types: + if label in known_types and label not in issue["type"]: issue["type"].append(str(label)) # creates an event for type updates and adds it to the eventsList @@ -634,10 +759,14 @@ def reformat_events(issue_data): issue["eventsList"].append(resolution_event) elif event["event"] == "commented": - # "state_new" and "resolution" of the issue give the information about the state and the resolution of + # "state_new" of the issue gives the information about the state of # the issue when the comment was written, because the eventsList is sorted by time event["event_info_1"] = issue["state_new"] - event["event_info_2"] = issue["resolution"] + # if event is a review comment, it can contain suggestions + if "contains_suggestion" in event: + event["event_info_2"] = str(event["contains_suggestion"]) + else: + event["event_info_2"] = str(False) elif event["event"] == "referenced" and event["commit"] is not None: # remove "referenced" events originating from commits @@ -651,7 +780,7 @@ def reformat_events(issue_data): for event_to_remove in events_to_remove: issue["eventsList"].remove(event_to_remove) - return issue_data + return def insert_user_data(issues, conf, resdir): @@ -662,7 +791,6 @@ def insert_user_data(issues, conf, resdir): :param issues: the issues to retrieve user data from :param conf: the project configuration :param resdir: the directory in which the username-to-user-list should be dumped - :return: the updated issue data """ log.info("Syncing users with ID service...") @@ -692,7 +820,7 @@ def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames= # ensure string representation for name and e-mail address username = str(user["username"]) - name = str(user["name"]) if "name" in user else username + name = str(user["name"]) if user["name"] is not None else username mail = str(user["email"]) # construct string for ID service and send query @@ -750,6 +878,9 @@ def get_user_from_id(idx, buffer_db=user_buffer): for event in issue["eventsList"]: event["user"] = get_id_and_update_user(event["user"]) + if event["event"] == "commit_added": + event["event_info_2"] = get_id_and_update_user(event["event_info_2"]) + # check database for the reference-target user if needed if event["ref_target"] != "": event["ref_target"] = get_id_and_update_user(event["ref_target"]) @@ -763,6 +894,10 @@ def get_user_from_id(idx, buffer_db=user_buffer): for event in issue["eventsList"]: event["user"] = get_user_from_id(event["user"]) + # for commit_added events, save the commit's author's name in event_info_2 + if event["event"] == "commit_added": + event["event_info_2"] = get_user_from_id(event["event_info_2"])["name"] + # get the reference-target user if needed if event["ref_target"] != "": event["ref_target"] = get_user_from_id(event["ref_target"]) @@ -773,17 +908,18 @@ def get_user_from_id(idx, buffer_db=user_buffer): lines = [] for username in username_id_buffer: user = get_user_from_id(username_id_buffer[username]) - lines.append(( - username, - user["name"], - user["email"] - )) + if not username == "None": + lines.append(( + username, + user["name"], + user["email"] + )) log.info("Dump username list to file...") username_dump = os.path.join(resdir, "usernames.list") csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) - return issues + return def print_to_disk(issues, results_folder): @@ -810,7 +946,7 @@ def print_to_disk(issues, results_folder): json.dumps(issue["resolution"]), issue["created_at"], issue["closed_at"], - json.dumps([]), # components + json.dumps(issue["subIssues"]), # components event["event"], event["user"]["name"], event["user"]["email"], diff --git a/issue_processing/jira_issue_processing.py b/issue_processing/jira_issue_processing.py index 4220b96..9516ea3 100644 --- a/issue_processing/jira_issue_processing.py +++ b/issue_processing/jira_issue_processing.py @@ -19,6 +19,8 @@ # Copyright 2020-2021 by Thomas Bock # Copyright 2026 by Thomas Bock # Copyright 2023, 2025 by Maximilian Löffler +# Copyright 2025-2026 by Leo Sendelbach +# Copyright 2025-2026 by Ritika Hiremath # All Rights Reserved. """ This file is able to extract Jira issue data from xml files. @@ -37,7 +39,7 @@ from codeface_utils.cluster.idManager import dbIdManager, csvIdManager from codeface_utils.configuration import Configuration from codeface_utils.dbmanager import DBManager - +from codeface_utils.util import setup_logging from csv_writer import csv_writer from jira import JIRA @@ -128,7 +130,7 @@ def run(): referenced_issue["history"].append(referenced_by) # 5) update user data with Codeface database - processed_issues = insert_user_data(processed_issues, __conf) + insert_user_data(processed_issues, __conf) # 6) dump result to disk print_to_disk(processed_issues, __resdir) # # 7) export for Gephi @@ -303,9 +305,12 @@ def parse_xml(issue_data, persons, skip_history, referenced_bys): link = issue_x.getElementsByTagName("link")[0] issue["url"] = link.firstChild.data - type = issue_x.getElementsByTagName("type")[0] - issue["type"] = type.firstChild.data - issue["type_list"] = ["issue", str(type.firstChild.data.lower())] + type = issue_x.getElementsByTagName("type")[0].firstChild.data + # rename 'new feature' type to 'feature' to be in line with the github original issue type + if type == "New Feature": + type = "Feature" + issue["type"] = type + issue["type_list"] = ["issue", str(type.lower())] status = issue_x.getElementsByTagName("status")[0] issue["state"] = status.firstChild.data @@ -463,21 +468,19 @@ def load_issues_via_api(issues, persons, url, referenced_bys): for change in changelog.histories: # default values for state and resolution - old_state, new_state, old_resolution, new_resolution = "open", "open", "unresolved", "unresolved" + new_state, old_resolution, new_resolution = "open", "unresolved", "unresolved" # all changes in the issue changelog are checked if they contain a useful information for item in change.items: # state_updated event gets created and added to the issue history if item.field == "status": - if item.fromString is not None: - old_state = item.fromString.lower() if item.toString is not None: new_state = item.toString.lower() history = dict() history["event"] = "state_updated" history["event_info_1"] = new_state - history["event_info_2"] = old_state + history["event_info_2"] = "" if hasattr(change, "author"): user = create_user(change.author.displayName, change.author.name, "") else: @@ -611,7 +614,7 @@ def get_user_string(name, email): def get_id_and_update_user(user, buffer_db_ids=user_id_buffer): # ensure string representation for name and e-mail address - name = str(user["name"]) if "name" in user else str(user["username"]) + name = str(user["name"]) if user["name"] is not None else str(user["username"]) mail = str(user["email"]) # may be empty # construct string for ID service and send query @@ -692,7 +695,7 @@ def get_user_from_id(idx, buffer_db=user_buffer): event["event_info_2"] = assigned_user["email"] log.debug("number of issues after insert_user_data: '{}'".format(len(issues))) - return issues + return def print_to_disk(issues, results_folder): diff --git a/issue_processing/zulip_issue_processing.py b/issue_processing/zulip_issue_processing.py new file mode 100644 index 0000000..926d021 --- /dev/null +++ b/issue_processing/zulip_issue_processing.py @@ -0,0 +1,663 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2017 by Raphael Nömmer +# Copyright 2017 by Claus Hunsen +# Copyright 2018 by Barbara Eckl +# Copyright 2018-2019 by Anselm Fehnker +# Copyright 2019 by Thomas Bock +# Copyright 2020-2021 by Thomas Bock +# Copyright 2026 by Thomas Bock +# Copyright 2025 by Maximilian Löffler +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. +""" +This file is able to extract Zulip issue data from json files. +""" + +import argparse +import json +import os +import sys +from datetime import datetime, timedelta +from logging import getLogger + +from codeface_utils.cluster.idManager import dbIdManager, csvIdManager +from codeface_utils.configuration import Configuration +from codeface_utils.dbmanager import DBManager +from codeface_utils.util import setup_logging +from dateutil import parser as dateparser +from bs4 import BeautifulSoup + +from csv_writer import csv_writer + +# create logger +setup_logging() +log = getLogger(__name__) + +# datetime format string +datetime_format = "%Y-%m-%d %H:%M:%S" + +def run(): + # get all needed paths and arguments for the method call. + parser = argparse.ArgumentParser(prog='codeface-extraction-issues-github', description='Codeface extraction') + parser.add_argument('-c', '--config', help="Codeface configuration file", default='codeface.conf') + parser.add_argument('-p', '--project', help="Project configuration file", required=True) + parser.add_argument('resdir', help="Directory to store analysis results in") + + # parse arguments + args = parser.parse_args(sys.argv[1:]) + __codeface_conf, __project_conf = list(map(os.path.abspath, (args.config, args.project))) + + # create configuration + __conf = Configuration.load(__codeface_conf, __project_conf) + + # get source and results folders + __srcdir = os.path.abspath(os.path.join(args.resdir, __conf['repo'] + "_issues")) + __resdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) + __userdir = os.path.abspath(os.path.join(args.resdir, __conf['project'], __conf["tagging"])) + # run processing of issue data: + # 1) load the list of issues + issues = load(__srcdir) + log.info("Source file loaded") + users = load_users(__userdir) + # 2) update missing columns + issues = update(issues, users) + # 3) re-format the issues + issues = reformat_issues(issues) + # 5) update user data with Codeface database and dump username-to-name/e-mail list + issues = insert_user_data(issues, __conf, __resdir) + # 6) dump result to disk + print_to_disk(issues, __resdir) + log.info("Zulip issue processing complete!") + + +def load(source_folder): + """Load issues from disk. + + :param source_folder: the folder where to find 'zulip.json' + :return: the loaded zulip data + """ + + srcfile = os.path.join(source_folder, "zulip.json") + log.info("Loading Zulip data from file '{}'...".format(srcfile)) + + # check if file exists and exit early if not + if not os.path.exists(srcfile): + log.error("Zulip data file '{}' does not exist! Exiting early...".format(srcfile)) + sys.exit(-1) + + with open(srcfile) as issues_file: + issue_data = json.load(issues_file) + + return issue_data + +def load_users(source_folder): + """Load users list from disk if it exists. + + :param source_folder: the folder where to find 'usernames.list' + :return: the loaded zulip data + """ + + srcfile = os.path.join(source_folder, "usernames.list") + log.info("Loading users data from file '{}'...".format(srcfile)) + + # check if file exists and exit early if not + if not os.path.exists(srcfile): + log.error("Users data file '{}' does not exist! Continuing without...".format(srcfile)) + return {} + + users = {} + # cleans and opens the source file + with open(srcfile, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + + parts = line.split(";") + if len(parts) != 3: + log.warning(f"Skipping malformed line: {line}") + continue + + username, name, email = [p.strip('"') for p in parts] + + # creates access to users data through username unless it is None + if username != "None": + users[username] = {"name": name, "email": email} + + # creates access to users data through name + users[name] = { + "username": "" if username == "None" else username, + "email": email, + } + + log.info(f"Loaded {len(users)} users") + return users + +def format_time(time): + """ + Format times from different sources to a consistent time format + + :param time: the time that shall be formatted + :return: the formatted time + """ + + # empty time would be formatted to current date + if time == "" or time is None: + return "" + else: + d = datetime.fromtimestamp(time) + return d.strftime(datetime_format) + + +def subtract_seconds_from_time(time, seconds): + """ + Subtract the specified number of seconds from a date string + + :param time: the date string to subtract the specified seconds from + :param seconds: the number of seconds to subtract from the date string + :return: the date string after subtracting the specified number of seconds + """ + + new_time = datetime.strptime(time, datetime_format) - timedelta(seconds = seconds) + return new_time.strftime(datetime_format) + + +def create_user(name, username, email): + """ + Creates a user object with all needed information + + :param name: the name the user shall have + :param username: the username the user shall have + :param email: the email the user shall have + :return: the created user object + """ + + if name is None: + name = "" + if username is None: + username = "" + if email is None: + email = "" + + user = dict() + user["username"] = username + user["name"] = name + user["email"] = email + + return user + + +def create_deleted_user(): + """ + Creates a user object for a deleted user (ghost user) + + :return: the created user object for a deleted user + """ + + return create_user("Deleted user", "ghost", "ghost@github.com") + + +def lookup_user(user_dict, user): + """ + Alters a user object in the case that name or email are missing by the corresponding name and email + from a given user dictionary + + :param user_dict: the user dictionary + :param user: the user object to lookup in the dictionary + :return: the altered user object in case of a lookup + or the unaltered user object otherwise + """ + + # if user is None, replace it by a deleted user + if user is None: + user = create_deleted_user() + + if (user["name"] == "" or user["name"] is None or + user["email"] is None or user["email"] == ""): + + # lookup user only if username is not None and not empty + if user["username"] is not None and not user["username"] == "": + user = user_dict[user["username"]] + + return user + +def update_user_dict(user_dict, user): + """ + Adds or updates users to merge GitHub usernames and names and e-mail addresses originating from the git repository + + :param user_dict: the user dictionary + :param user: the user object to add to or update in the dictionary + :return: the updated user dictionary + """ + + # if the given user is None, use the deleted user instead + if user is None: + user = create_deleted_user() + + if user["username"] not in list(user_dict.keys()): + if user["username"] is not None and not user["username"] == "": + user_dict[user["username"]] = user + else: + user_in_dict = user_dict[user["username"]] + if user_in_dict["name"] is None or user_in_dict["name"] == "": + user_in_dict["name"] = user["name"] + if user_in_dict["email"] is None or user_in_dict["email"] == "": + user_in_dict["email"] = user["email"] + user_dict[user["username"]] = user_in_dict + + return user_dict + +def discussion_id_update(issue_data): + """ + Updates the id for each issue data in zulip. + The ID is dependent on the topic id and followed by # and + the count of the message from the beginning. + :param issue_data: total issue data from zulip + : return: The updated issue data with the id updated. + """ + grouped = {} + # groups each discussion topic together to update the id + for item in issue_data: + topic = item["discussion_topic"] + if not topic: + log.warning("Missing discussion_topic in item:" + str(item)) + continue + if topic not in grouped: + grouped[topic] = [] + log.debug("New topic group created:" + str(topic)) + grouped[topic].append(item) + + # Updates the discussion id here. + for topic, messages in grouped.items(): + messages.sort(key=lambda m: m["timestamp"]) + for idx, msg in enumerate(messages, start=1): + msg["discussion_id"] = f'{msg["discussion_id"]}#{idx}' + + log.info("Finished updating discussion id for all issue data") + return issue_data + +def discussion_begin_end_add(issue_data): + """ + Updates the discussion begin and end time for each discussion topic. + :param issue_data: Total issue data + :return: the updated issue data with two new columns. + """ + + discussion_topics = {} + # groups discussion topics and finds the max and min time that topic was discussed. + for item in issue_data: + d_topic = item["discussion_topic"] + ts = item["timestamp"] + + if d_topic not in discussion_topics: + discussion_topics[d_topic] = [] + log.debug("New discussion topic found:" + str(d_topic)) + discussion_topics[d_topic].append(ts) + + if not discussion_topics: + log.warning("No discussion topics were grouped!") + + discussion_bounds = { + d_topic: { + "discussion_begin": min(times), + "discussion_end": max(times) + } + for d_topic, times in discussion_topics.items() + } + + # for d_topic, bounds in discussion_bounds.items(): + # log.debug("Topic" + str(d_topic) + "->" + str(bounds)) + + for item in issue_data: + d_topic = item["discussion_topic"] + item["discussion_begin"] = discussion_bounds[d_topic]["discussion_begin"] + item["discussion_end"] = discussion_bounds[d_topic]["discussion_end"] + log.debug("Discussion bounds updated for "+ str(item)) + + log.info("Finished updating discussion bounds") + return issue_data + +def bot_event_type(issue): + """ + Updates the type of event . + This function only updates events notification bot username "Notification Bot". + + :param issue: the zulip issue data to update + :return: the event type + """ + # extracts the content into a variable + content = issue.get("content", "").lower() + # checks if the event type is present in the string + if "stream created" in content: + return "stream created" + + if "changed the description" in content: + return "stream description changed" + + if "changed the access permissions" in content: + return "stream permissions changed" + + if "renamed stream" in content: + return "stream renamed" + + if "wave" in content: + return "wave" + + return "unclassified event" + +def notification_bot_event(issue): + """ + Updates the type of event . + This function only updates events notification bot does with no change in username. + + :param issue: the zulip issue data to update + :return: the event type + """ + # extracts the content into a variable + content = issue.get("content", "").lower() + # checks if the event type is present in the string + if "has marked this topic as resolved" in content: + return "topic resolved" + + if "has marked this topic as unresolved" in content: + return "topic unresolved" + + if "topic was moved" in content: + return "topic moved" + + return "user event" + +def bot_event_name_update(issue): + """ + For events from notification bot. + It finds the user in content string and returns it + + :param issue: a single zulip-issue data to find the user name for. + :return: returns the user name + """ + soup = BeautifulSoup(issue["content"], "html.parser") + mention = soup.find("span", class_="user-mention") + + if mention: + return mention.get_text(strip=True) + + return None + +def create_update_user(issue, users): + """ + Creates user for each issue data. + Classifies name and username based on whethre the name has a space or not. + + :param issue: A single issue data from the zulip issue data + :reutrn: returns a dictionary to update issue["user"] + """ + log.debug("Creating user for issue " + str(issue["discussion_id"])) + + dict_issue = {} + sender = issue["sender_full_name"] + + # check if present in users list + if sender in users: + info = users[sender] + + dict_issue["username"] = info.get("username", sender) + dict_issue["name"] = info.get("name", sender) + dict_issue["email"] = info.get("email", issue["sender_email"]) + + else: + # fallback + # checks if there is a space in the string sender. + if " " in sender: + dict_issue["username"] = "" + dict_issue["name"] = sender + + # if it does not find a name then updates both with sa + else: + dict_issue["username"] = sender + dict_issue["name"] = sender + + dict_issue["email"] = issue["sender_email"] + + # log.debug("New User dict created for" + str(issue["discussion_topic"]) + "with" + str(issue["discussion_id"])) + return dict_issue + +def event_type_and_user(issue_data, username): + """ + Checks if the event is a stream events. + updates the event type and sender details, if sender name is made into notification bot. + :param issue_data: total zulip issue data to check and update the event types. + :return: returns the updated zulip issue data + """ + + for issue in issue_data: + if(("stream events" in issue["discussion_topic"]) and (issue["sender_full_name"] == "Notification Bot")): + # log.debug("Bot stream event detected") + # updates name and discussion + issue["individual_events"]= bot_event_type(issue) + issue["sender_full_name"] = bot_event_name_update(issue) + soup = BeautifulSoup(issue["content"], "html.parser") + span_data = soup.find("span", class_="user-mention") + if span_data: + user_id = span_data.get("data-user-id") + issue["sender_email"] = user_id+"@zulipchat.com" + + else: + if("stream events" in issue["discussion_topic"]): + # log.debug("User stream event detected") + # updates the event type when issue name is proper user name. + issue["individual_events"] = notification_bot_event(issue) + + issue["individual_events"]= "commented event" + # creates user for each issue data. Combines name, email and username into a dictionary. + issue["user"] = create_update_user(issue, username) + + log.info("Finished updating event types and users") + return issue_data + +def update(issue_data, users): + """ + updates values in the issue data as per requirement. + :params: issue_data: the issue data to be updated. + :params: usernames: existing list of user data + :return: returns the issue data. + """ + + # sends the entirety of the issue data to update discussion id, discussion begin, discussion end, event type, and user. + issue_data = discussion_id_update(issue_data) + issue_data = discussion_begin_end_add(issue_data) + issue_data = event_type_and_user(issue_data,users) + + log.info("Issue data updated ...") + return issue_data + +def reformat_issues(issue_data): + """ + Re-arrange issue data structure. + + :param issue_data: the issue data to re-arrange + :return: the re-arranged issue data + """ + + log.info("Re-arranging Github issues...") + + # re-process all issues + for issue in issue_data: + + # empty container for issue types + issue["type"] = [] + + # empty container for issue resolutions + issue["resolution"] = [] + + issue["discussion_begin"] = format_time(issue["discussion_begin"]) + + # parses the close time in the correct format + issue["discussion_end"] = format_time(issue["discussion_end"]) + + issue["timestamp"] = format_time(issue["timestamp"]) + + issue["type"].append("topic") + + return issue_data + +def insert_user_data(issues, conf, resdir): + """ + Insert user data into database and update issue data. + In addition, dump username-to-user list to file. + + :param issues: the issues to retrieve user data from + :param conf: the project configuration + :param resdir: the directory in which the username-to-user-list should be dumped + :return: the updated issue data + """ + + log.info("Syncing users with ID service...") + + # create buffer for users (key: user id) + user_buffer = dict() + # create buffer for user ids (key: user string) + user_id_buffer = dict() + # create buffer for usernames (key: username) + username_id_buffer = dict() + + # connect to ID service + if conf["useCsv"]: + idservice = csvIdManager(conf) + else: + dbm = DBManager(conf) + idservice = dbIdManager(dbm, conf) + + def get_user_string(name, email): + if not email or email is None: + return "{name}".format(name=name) + # return "{name} <{name}@default.com>".format(name=name) # for debugging only + else: + return "{name} <{email}>".format(name=name, email=email) + + def get_id_and_update_user(user, buffer_db_ids=user_id_buffer, buffer_usernames=username_id_buffer): + + # ensure string representation for name and e-mail address + username = str(user["username"]) + name = str(user["name"]) if user["name"] is not None else username + mail = str(user["email"]) + + # construct string for ID service and send query + user_string = get_user_string(name, mail) + + # check buffer to reduce amount of DB queries + if user_string in buffer_db_ids: + log.info("Returning person id for user '{}' from buffer.".format(user_string)) + if username is not None: + buffer_usernames[username] = buffer_db_ids[user_string] + return buffer_db_ids[user_string] + + # get person information from ID service + log.info("Passing user '{}' to ID service.".format(user_string)) + idx = idservice.getPersonID(user_string) + + # add user information to buffer + # user_string = get_user_string(user["name"], user["email"]) # update for + buffer_db_ids[user_string] = idx + + # add id to username buffer + if username is not None: + buffer_usernames[username] = idx + + return idx + + def get_user_from_id(idx, buffer_db=user_buffer): + + # check whether user information is in buffer to reduce amount of DB queries + if idx in buffer_db: + log.info("Returning user '{}' from buffer.".format(idx)) + return buffer_db[idx] + + # get person information from ID service + log.info("Passing user id '{}' to ID service.".format(idx)) + person = idservice.getPersonFromDB(idx) + user = { + "name": person["name"], + "email": person["email1"], + "id": person["id"] + } + + # add user information to buffer + buffer_db[idx] = user + + return user + + + # check and update database for all occurring users + for issue in issues: + # check database for issue author + issue["user"] = get_id_and_update_user(issue["user"]) + + # get all users after database updates having been performed + for issue in issues: + # get issue author + issue["user"] = get_user_from_id(issue["user"]) + + # dump username, name, and e-mail to file + lines = [] + for username in username_id_buffer: + user = get_user_from_id(username_id_buffer[username]) + lines.append(( + username, + user["name"], + user["email"] + )) + + log.info("Dump username list to file...") + username_dump = os.path.join(resdir, "usernames.list") + csv_writer.write_to_csv(username_dump, sorted(set(lines), key=lambda line: line[0])) + + return issues + +def print_to_disk(issues, results_folder): + """ + Print issues to file "issues-zulip.list" in the results folder. + + :param issues: the issues to dump + :param results_folder: the folder where to place "issues-zulip.list" output file + """ + + # construct path to output file + output_file = os.path.join(results_folder, "issues-zulip.list") + log.info("Dumping output in file '{}'...".format(output_file)) + + # construct lines of output + lines = [] + for issue in issues: + # print(issue["user"]) + lines.append(( + issue["discussion_id"], + issue["discussion_topic"], + json.dumps(issue["type"]), + json.dumps([]), + json.dumps(issue["resolution"]), + issue["discussion_begin"], + issue["discussion_end"], + json.dumps([]), # components + issue["individual_events"], + issue["user"]["name"], + issue["user"]["email"], + issue["timestamp"], + json.dumps([]), + json.dumps([]) + )) + + # write to output file + csv_writer.write_to_csv(output_file, sorted(set(lines), key=lambda line: lines.index(line))) \ No newline at end of file diff --git a/run-combine-projects.py b/run-combine-projects.py new file mode 100644 index 0000000..0646b57 --- /dev/null +++ b/run-combine-projects.py @@ -0,0 +1,19 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2026 by Ritika Hiremath +# All Rights Reserved. +from combine_projects import combine_projects + +combine_projects.run() diff --git a/run-zulip-data-extraction.py b/run-zulip-data-extraction.py new file mode 100644 index 0000000..9827220 --- /dev/null +++ b/run-zulip-data-extraction.py @@ -0,0 +1,20 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. +from zulip_data_extraction import zulip_data_extraction as scraping + +scraping.run() + diff --git a/run-zulip-issues.py b/run-zulip-issues.py new file mode 100644 index 0000000..98f0725 --- /dev/null +++ b/run-zulip-issues.py @@ -0,0 +1,19 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. +import issue_processing.zulip_issue_processing as zulip_issues + +zulip_issues.run() diff --git a/zulip_data_extraction/__init__.py b/zulip_data_extraction/__init__.py new file mode 100644 index 0000000..9bad579 --- /dev/null +++ b/zulip_data_extraction/__init__.py @@ -0,0 +1 @@ +# coding=utf-8 diff --git a/zulip_data_extraction/zulip_data_extraction.py b/zulip_data_extraction/zulip_data_extraction.py new file mode 100644 index 0000000..ca93860 --- /dev/null +++ b/zulip_data_extraction/zulip_data_extraction.py @@ -0,0 +1,199 @@ +# coding=utf-8 +# This file is part of codeface-extraction, which is free software: you +# can redistribute it and/or modify it under the terms of the GNU General +# Public License as published by the Free Software Foundation, version 2. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Copyright 2025-2026 by Ritika Hiremath +# All Rights Reserved. +""" +This file is able to extract issue data from Zulip. +""" +import zulip +import json +import argparse +import time +import os +import sys +from codeface_utils.util import setup_logging +from logging import getLogger + + +# create logger +setup_logging() +log = getLogger(__name__) + +# Log in to https://rust-lang.zulipchat.com, go to Personal Settings -> Account&Privacy -> API key, and download the .zuliprc file or copy the API key and utilize the template. +# Template of zuliprc.txt is present in this directory. + +# The location of zuliprc.txt file +parser = argparse.ArgumentParser() +parser.add_argument("--zulip-config", default=None, help="Path to zuliprc config file") +parser.add_argument("--output", default=None, help="Path to the output directory") +args = parser.parse_args(sys.argv[1:]) + +# Resolve path +if args.zulip_config: + config_path = args.zulip_config +else: + config_path = os.path.join(os.path.dirname(__file__), "zuliprc") + +if args.output: + output_path = os.path.join(args.output, "zulip.json") +else: + output_path = os.path.join(os.path.dirname(__file__), "zulip.json") + +# Raise error if file not found +if not os.path.exists(config_path): + raise FileNotFoundError(f"Zulip config file not found at: {config_path}") + +client = zulip.Client(config_file= config_path) + + +def run(): + log.info("Starting Zulip data extraction") + # use exisiting zulip_streams_and_topics.json file + if os.path.exists(os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json")): + streams_and_topics_path = os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + with open(streams_and_topics_path, "r", encoding="utf-8") as f: + streams_and_topics = json.load(f) + + else: + streams_and_topics = topics_extraction() + messages_extraction_for_each_stream(streams_and_topics) + +def safe_get_topics(stream_id): + """ + Fetches all the topics from Zulip rust. + + :params: stream_id : id of the stream. + :return: list of topics in the stream. If no topics exist then returns empty list. + + """ + while True: + resp = client.get_stream_topics(stream_id = stream_id) + if resp["result"] == "success": + return [t["name"] for t in resp["topics"]] + elif resp["result"] == "error" and resp.get("code") == "RATE_LIMIT_HIT": + retry = int(resp.get("retry_after", 5)) + log.info("Rate limit hit, retrying in '{}'s...".format(retry)) + time.sleep(retry) + else: + log.error("Error fetching topics: '{}'", resp) + return [] + + +def topics_extraction(): + """ + Fetches Extract all streams and topics + + :return: returns a dictory of topics in the stream. + """ + streams = client.get_streams()["streams"] + data = {} + + for i, s in enumerate(streams, 1): + stream_name = s["name"] + log.debug(f"[{i}/{len(streams)}] Getting topics for: {stream_name}") + + topics = safe_get_topics(s["stream_id"]) + data[stream_name] = { + "topics": topics + } + + time.sleep(0.5) + + topics_file = os.path.join(os.path.dirname(output_path), "zulip_streams_and_topics.json") + with open(topics_file, "w", encoding = "utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + log.info("Saved zulip_streams_and_topics.json") + return data + + +def load_stream_topics(file_path): + """ + Fetches all the streams and topics from the file. + + :params: file_path : path of the file for streams and topics. + :return: returns the content of the file. + """ + with open(file_path, "r", encoding = "utf-8") as f: + return json.load(f) + + + +def fetch_all_messages_for_stream(stream_name,topic_name): + """ + Fetches all the messages in a topic through API call. + + :params: topic_name : name of the topic for which all. the messages should be fetched. + :return: returns all the messages from that topic. + """ + + all_messages = [] + anchor = 0 # Start from oldest + + while True: + request = { + "anchor": anchor, + "num_before": 0, + "num_after": 500, # max allowed + "narrow": [ + {"operator": "stream", "operand": stream_name}, + {"operator": "topic", "operand": topic_name} + ] + } + + resp = client.get_messages(request) + msgs = resp["messages"] + + if not msgs: + break + + all_messages.extend(msgs) + + # Move anchor forward for next batch + anchor = msgs[-1]["id"] + 1 + + time.sleep(0.4) + + return all_messages + + + +def messages_extraction_for_each_stream(streams_with_topics): + final_output = [] + + for stream_name, info in streams_with_topics.items(): + topics = info["topics"] + + for topic in topics: + log.debug(f"\n Fetching all messages for stream: {stream_name} and topic: {topic}") + + msgs = fetch_all_messages_for_stream(stream_name,topic) + + for m in msgs: + final_output.append({ + "discussion_id": m["stream_id"], + "discussion_topic": m["subject"], + "sender_full_name": m["sender_full_name"], + "sender_email": m["sender_email"], + "stream": stream_name, + "content": m["content"], + "timestamp": m["timestamp"] + }) + + # Save everything + with open(output_path, "w", encoding = "utf-8") as f: + json.dump(final_output, f, indent = 2) + + log.info("\n Saved all stream messages to: '{}'".format(output_path)) \ No newline at end of file diff --git a/zulip_data_extraction/zuliprc.txt b/zulip_data_extraction/zuliprc.txt new file mode 100644 index 0000000..37d2142 --- /dev/null +++ b/zulip_data_extraction/zuliprc.txt @@ -0,0 +1,4 @@ +[api] +email= +key= +site=https://rust-lang.zulipchat.com