sum_test.py

import os 
import openai
import json
import tiktoken
from joblib import Memory 
from pdb import set_trace as st
openai.api_key = os.environ["OPENAI_API_KEY"]
memory = Memory('cachedir', verbose=0)

@memory.cache
def llm(prompt, model_name, temperature=0.08, max_tokens=128, stop=None):
    if model_name.startswith("text-davinci") or ('instruct' in model_name):
        response = openai.Completion.create(
            model=model_name,
            prompt=prompt,
            temperature=temperature,
            max_tokens=max_tokens,
            top_p=1,
            frequency_penalty=0.0,
            presence_penalty=0.0,
            stop=stop,
        )
        return response["choices"][0]["text"]
    else:
        response = openai.ChatCompletion.create(
            model=model_name,
            messages = [{'role': 'user', 'content': prompt}],
            temperature=temperature,
            max_tokens=max_tokens,
            stop=stop,
        )
    return response["choices"][0]['message']['content']


sum_prompt = """You are an autonomous intelligent agent tasked with navigating a web browser. You will be provided with the following:
1. a snippet of the current web page's accessibility tree: a simplified representation of the webpage with key information.
2. the curent web pages' URL. 
Given the information above, please generate a 'state summarization' following these rules:
If the URL match the formats below, answer correspondingly: 
1. 'http://<your_server>/user/<username>', answer: You are on the profile of a user.
2. 'http://<your_server>/user/<username>/<subpage>', answer: You are on the <subpage> page of a user.
3. 'http://<your_server>/f/<forum name>', answer: You are on the main page of a forum. 
4. 'http://<your_server>/f/<forum name>/<subpage>', answer: You are on the <subpage> page of a forum. 
5. 'http://<your_server>/f/<forum name>/<any number>/<anything>', answer: You are on the page of a post. 
6. 'http://<your_server>/search?q=<searchquery>', answer: You are on the search result page. 
For other cases, denote the <page name> in beginning of the accessibility tree in "RootWebArea '<page name>' focused: True" as <page name>. 
Directly answer: You are on the <page name> page. 
Here are the inputs: 
"""

relabel_prompt = """You are an autonomous intelligent agent tasked with navigating a web browser.
You will be provided with the following information: 
1. A list of context summarizations you have seen in the past. 
2. a snippet of the current web page's accessibility tree: a simplified representation of the webpage with key information and the curent web pages' URL. 
Please generate the summarization of the current observation after 'SUMMARIZATION:'. 
Here are some requirements:
Requirement 1: Different types of webpages should have clearly different summarizations. For example, for GitHub there can be the main page of GitHub, the overview page of a GitHub user, the issues page of a GitHub repository, the search result page of GitHub. etc, you should clearly categorize those and make sure not to mix them up.
Requirement 2: Important: The summarization should be general, and concise, without any user/object/task specific information, instead, websites that fall into the same categories should have the same summarization, for example, the main page of every reddit forum should be categoried as the same context summarization: On the main page of a Reddit forum. You should never include the specific name of the forum in the summarization.
Requirement 3: The URLs will be very useful for you to determine the summarization.
Requirement 4: If the context is the same as one from the seen list, directly copy the best matching one word by word. 

Example 1:
Observation:
The current web page's accessibility tree:
Tab 0 (current): Postmill

[1] RootWebArea 'Postmill' focused: True
	[31] HeaderAsNonLandmark ''
		[32] link 'Home'
	[55] link 'Forums'
	[56] link 'Wiki'
	[64] searchbox 'Search query'
	[65] link 'Notifications (0)'
	[66] link 'Submit'
	[12] button 'MarvelsGrantMan136' focused: True hasPopup: menu expanded: True
	[243] link ' Profile'
	[239] link ' My account'
	[245] link ' User settings'
	[255] link ' Block list'
	[286] separator '' orientation: horizontal
	[270] button ' Dark mode'
URL: http://reddit.com/
SUMMARIZATION: On the main page of a reddit-like forum.

Example 2:
Observation:
The current web page's accessibility tree:
Tab 0 (current): Notice: Virtual Meetup for Harry Potter Enthusiasts!

[5778] RootWebArea 'Notice: Virtual Meetup for Harry Potter Enthusiasts!' focused: True
	[5820] HeaderAsNonLandmark ''
		[5821] link 'Home'
			[5860] StaticText 'Postmill'
	[5861] link 'Forums'
	[5862] link 'Wiki'
	[5870] searchbox 'Search query'
	[5871] link 'Notifications (0)'
	[5872] link 'Submit'
	[5789] button 'MarvelsGrantMan136' hasPopup: menu expanded: False
	[5791] main ''
		[6050] link '/f/books'
		[5792] article ''
			[5795] HeaderAsNonLandmark ''
				[5879] heading 'Notice: Virtual Meetup for Harry Potter Enthusiasts!'
					[6051] link 'Notice: Virtual Meetup for Harry Potter Enthusiasts!'
				[6053] StaticText 'Submitted by '
				[5798] link 'MarvelsGrantMan136' expanded: False
				[6055] time 'January 13, 2024 at 12:04:20 PM EST'
					[6299] StaticText '0 seconds ago'
				[6056] StaticText ' in '
				[6057] link 'books'
			[5882] StaticText 'Hi Everyone, Kindly notice that there will be a virtual meetup for Harry Potter enthusiasts on July 8th.'
			[6302] link 'No comments'
			[6303] link 'Edit'
			[6334] button 'Delete'
			[5886] button 'Retract upvote'
			[6062] StaticText '1'

URL: http://reddit.com/f/books/19/notice-virtual-meetup-for-harry-potter-enthusiasts
SUMMARIZATION: On the page of a Reddit post.

Example 3:
Observation:
The current web page's accessibility tree:
Tab 0 (current): Editing biography for MarvelsGrantMan136

[14346] RootWebArea 'Editing biography for MarvelsGrantMan136' focused: True
	[14373] HeaderAsNonLandmark ''
		[14374] link 'Home'
			[14403] StaticText 'Postmill'
	[14404] link 'Forums'
	[14405] link 'Wiki'
	[14413] searchbox 'Search query'
	[14414] link 'Notifications (0)'
	[14415] link 'Submit'
	[14357] button 'MarvelsGrantMan136' hasPopup: menu expanded: False
	[14364] main ''
		[14366] heading 'Editing biography for /u/MarvelsGrantMan136'
			[14388] link '/u/MarvelsGrantMan136'
		[14421] StaticText 'Biography'
		[14422] textbox 'Biography' required: False describedby: user_biography_biography_help
			[14549] StaticText 'random'
		[14436] heading 'Preview'
		[14424] checkbox 'Formatting help +' checked: false
		[14426] StaticText 'Markdown allowed.'
		[14440] button 'Save'
	[14365] complementary ''
		[14393] HeaderAsNonLandmark ''
			[14442] heading 'MarvelsGrantMan136'
				[14739] link 'MarvelsGrantMan136'
			[14742] StaticText 'Registered '
			[14743] time 'June 24, 2020 at 11:02:10 AM EDT'
				[14763] StaticText '4 years ago'
			[14746] StaticText 'Not whitelisted'
		[14445] StaticText 'random'
		[14446] link 'Edit biography'
		[14396] heading 'Toolbox'
		[14751] link 'Hidden forums'
		[14752] link 'Trash'
	[14361] contentinfo ''
		[14399] StaticText 'Running '
		[14400] link 'Postmill'

URL: http://reddit.com/user/MarvelsGrantMan136/edit_biography
SUMMARIZATION: On the biography edit page of a Reddit user.

Example 4:
Observation:
The current web page's accessibility tree:
Tab 0 (current): books

[3356] RootWebArea 'books' focused: True
	[3566] HeaderAsNonLandmark ''
		[3567] link 'Home'
			[3599] StaticText 'Postmill'
	[3600] link 'Forums'
	[3601] link 'Wiki'
	[3609] searchbox 'Search query'
	[3610] link 'Notifications (0)'
	[3611] link 'Submit'
	[3367] button 'MarvelsGrantMan136' hasPopup: menu expanded: False
	[3369] main ''
		[3559] heading '/f/books'
		[4039] link 'Submissions'
		[4040] link 'Comments'
		[3373] button 'Sort by: Hot' hasPopup: menu expanded: False
		[3374] article ''
			[3377] HeaderAsNonLandmark ''
				[3618] heading 'big little lies'
					[4042] link 'big little lies'
				[4044] StaticText 'Submitted by '
				[3380] link 'MarvelsGrantMan136' expanded: False
				[4046] time 'January 10, 2024 at 7:40:23 AM EST'
					[4529] StaticText '1 day ago'
			[4530] link 'No comments'
			[4532] link 'Edit'
			[5089] button 'Delete'
			[3622] button 'Retract upvote'
			[4051] StaticText '1'
			[3625] button 'Downvote'

URL: http://reddit.com/f/books
SUMMARIZATION: On the main page of a Reddit forum.

Example 5:
Observation:
The current web page's accessibility tree:
Tab 0 (current): All forums

[2803] RootWebArea 'All forums' focused: True
	[2829] HeaderAsNonLandmark ''
		[2830] link 'Home'
			[3036] StaticText 'Postmill'
	[3037] link 'Forums'
	[3038] link 'Wiki'
	[3046] searchbox 'Search query'
	[3047] link 'Notifications (0)'
	[3048] link 'Submit'
	[2814] button 'MarvelsGrantMan136' hasPopup: menu expanded: False
	[2821] main ''
		[3050] link 'Forums'
		[3051] link 'Tags'
		[3052] link 'Alphabetical'
		[2824] heading 'All forums'
		[2869] heading 'A'
		[3053] link 'allentown'
		[3054] link 'arlingtonva'
		[3055] link 'Art'
		[3056] link 'AskReddit'
		[3057] link 'askscience'
		[3058] link 'aww'
		[2878] heading 'B'
		[3059] link 'baltimore'
		[3060] link 'books'
		[3061] link 'boston'
		[3062] link 'BridgeportCT'
		[3063] link 'BuyItForLife'
		[2886] heading 'C'
		[3064] link 'CambridgeMA'
		[3067] link 'consoles'

URL: http://reddit.com/forums/all
SUMMARIZATION: On the All forums page of a Reddit-like Platform, with element RootWebArea 'All forums' at the beginning. 


Here are the inputs: 
1. Here is the list: 
{seen_list}

2. Here is the trajectory:
{traj}
"""


def is_profile(accessibility_tree):
    if not "] link 'Overview'" in accessibility_tree:
        return False 
    all_lines = accessibility_tree.split('\n')
    for i, line in enumerate(all_lines[:-2]):
        if "] link 'Overview'" in line and "] link 'Submissions'" in all_lines[i+1] and "] link 'Comments'" in all_lines[i+2]:
            return True
    return False


def format_input(obs, url, limit=5):
    obs = '\n'.join(obs.split('\n')[:limit])
    full_obs = f"The current web page's accessibility tree:\n{obs}\nURL: {url}"
    return full_obs


def get_status(obs, url, logger, method='hardcode'):
    if method == 'self':
        full_obs = format_input(obs, url, limit=30)
        file_path = './seen_states.json'
        with open(file_path, 'r') as file:
            seen_states = json.load(file)
        seen_list = list(list(seen_states.keys()))
        prompt = f"""{relabel_prompt.format(seen_list='* '.join(seen_list), traj=full_obs)}\nSUMMARIZATION:"""
        result = llm(prompt, 'gpt-4-1106-preview', max_tokens=64)
        result = result.lstrip()
        if result not in seen_list:
            logger.log(f"\033[35mNewly Added: \033[0m{result}")
            seen_states[result] = (0, 0)

            with open('./seen_states.json', 'w') as file:
                json.dump(seen_states, file, indent=4)
    return result