main.py

import argparse
import os
import requests
from bs4 import BeautifulSoup
import re
import pandas as pd
import json

# Extracting base url using argparse form CLI

parser = argparse.ArgumentParser()
parser.add_argument("--url", required=True, help="The base URL to scrape")
parser.add_argument("--out_dir", required=True, help="Directory to save the output files")
args = parser.parse_args()


os.makedirs(args.out_dir, exist_ok=True)


base_url = args.url

# Extraction of Homepage
r = requests.get(base_url)

soup = BeautifulSoup(r.text,'html.parser')

# Finding Search link
s=""
for links in soup.find_all("a"):
    if(links.get_text() == "Search"):
        s = links.get("href")

search_url = base_url + s

r2 = requests.get(search_url)

# Navigation to search page
soup = BeautifulSoup(r2.text,'html.parser')

# finding table tage and extracting all links from with help of anchor tag
table = soup.find('table')

table_links = []


if table:
    for a_tag in table.find_all('a', href=True):
        full_link = base_url + a_tag['href']
        table_links.append(full_link)

all_foa_data = []

# navgating to every page and extracting some fix details 

for i in range(len(table_links)):
    foa = requests.get(table_links[i])
        

    soup = BeautifulSoup(foa.text, 'html.parser')

    foa_data = {}

    
    title_tag = soup.find('h2', class_=re.compile(r'margin-bottom-0'))
    foa_data['Opportunity Title'] = title_tag.get_text(strip=True) if title_tag else None

    opp_num_tag = soup.find(string=re.compile(r'Funding opportunity number'))
    foa_data['Opportunity Number'] = opp_num_tag.find_next('div').get_text(strip=True) if opp_num_tag else None

    agency_tag = soup.find('span', string=re.compile(r'Agency:'))
    foa_data['Agency'] = agency_tag.parent.get_text(strip=True).replace('Agency:', '').strip() if agency_tag else None

    cfda_tag = soup.find('a', string=re.compile(r'Assistance Listings:'))
    foa_data['Assistance Listing (CFDA)'] = cfda_tag.parent.get_text(strip=True).replace('Assistance Listings:', '').strip() if cfda_tag else None

    # Description
    
    toggled_div = soup.find('div', attrs={'data-testid': 'toggled-content-container'})

    if toggled_div:
       
        visible_div = toggled_div.find_previous_sibling('div')
        part_1 = visible_div.get_text(strip=True) if visible_div else ""
       
        part_2 = toggled_div.get_text(strip=True)
        
        full_description = f"{part_1} {part_2}"

        foa_data['Description'] = " ".join(full_description.split())
    else:
        foa_data['Description'] = None
    
    
    if foa_data['Description'] is None:
        desc_container = soup.find('div', attrs={'data-testid': 'opportunity-description'})
        if desc_container:
        
          header_div = desc_container.find('div', class_=re.compile(r'display-block'))
    
          content_div = header_div.find_next_sibling('div') if header_div else None
          if content_div:
            raw_description = content_div.get_text(separator=' ', strip=True)
        
            foa_data['Description'] = ' '.join(raw_description.split())
          else:
             foa_data['Description'] = None
        

    close_date_tag = soup.find('strong', string=re.compile(r'Closing:'))
    foa_data['Close Date'] = close_date_tag.find_next('span').get_text(strip=True) if close_date_tag else None

    posted_date_tag = soup.find(lambda tag: tag.name == 'p' and 'Posted date' in tag.get_text())
    foa_data['Posted Date'] = posted_date_tag.find_next_sibling('p').get_text(strip=True) if posted_date_tag else None

    def get_award_stat(label):
        tag = soup.find('p', string=re.compile(label))
        if tag:
            sibling = tag.find_previous_sibling('p')
            return sibling.get_text(strip=True) if sibling else None
        return None

    foa_data['Total Program Funding'] = get_award_stat('Program Funding')
    foa_data['Expected Awards'] = get_award_stat('Expected awards')
    foa_data['Award Minimum'] = get_award_stat('Award Minimum')
    foa_data['Award Maximum'] = get_award_stat('Award Maximum')


    cost_sharing_requirment = soup.find(lambda tag: tag.name == 'p' and 'Cost sharing or matching requirement' in tag.get_text())
    foa_data['Cost Sharing Requirement'] = cost_sharing_requirment.find_next_sibling('div').get_text(strip=True) if cost_sharing_requirment else None

    funding_instrument_type = soup.find(lambda tag: tag.name == 'p' and 'Funding instrument type' in tag.get_text())
    foa_data['Funding Instrument Type'] = funding_instrument_type.find_next_sibling('div').get_text(strip=True) if funding_instrument_type else None


    category_of_funding = soup.find(lambda tag: tag.name == 'p' and 'Category of Funding Activity' in tag.get_text())
    foa_data['Category of Funding'] = category_of_funding.find_next_sibling('div').get_text(strip=True) if category_of_funding else None


    eligibility_header = soup.find('h3', string='Eligible applicants')
    eligibility_list = []
    if eligibility_header:
       
        for div in eligibility_header.find_next_siblings('div'):
            group_title_tag = div.find('h4')
            if not group_title_tag: 
                break 
            
            group_title = group_title_tag.get_text(strip=True)
            items = [li.get_text(strip=True) for li in div.find_all('li')]
            eligibility_list.append(f"{group_title}: {', '.join(items)}")

    foa_data['Eligible Applicants'] = " | ".join(eligibility_list) if eligibility_list else None


    add_info_header = soup.find('h3', string='Additional information')
    foa_data['Eligibility Additional Info'] = add_info_header.find_next_sibling('div').get_text(strip=True) if add_info_header else None



    contact_header = soup.find('h2', string=re.compile('Grantor contact information'))
    if contact_header:
        mailto = contact_header.find_next('a', href=re.compile(r'mailto:'))
        foa_data['Contact Email'] = mailto.get_text(strip=True) if mailto else None


    documents = []
    doc_table = soup.find('table')
    if doc_table:
        for row in doc_table.find('tbody').find_all('tr'):
            a_tag = row.find('a')
            if a_tag:
                documents.append({
                    "File Name": a_tag.get_text(strip=True),
                    "URL": a_tag['href']
                })
    foa_data['Documents'] = documents


    all_foa_data.append(foa_data)

# Creating out director to store csv and json files

json_output_path = os.path.join(args.out_dir, "foa.json")
csv_output_path = os.path.join(args.out_dir, "foa.csv")

with open(json_output_path, "w", encoding="utf-8") as f:
    json.dump(all_foa_data, f, indent=4)
    
df = pd.DataFrame(all_foa_data)

df.to_csv(csv_output_path, index=False, encoding="utf-8")

print(f"Finished! Successfully saved {len(all_foa_data)} FOAs to {json_output_path} and {csv_output_path}")