Skip to content

Use the model's template (ollama show --template <model>) #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mcelrath wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Use the model's template (ollama show --template <model>) #53

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 6 additions & 55 deletions python/complete.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,44 +17,7 @@
# create logger
log = None

def load_config(modelname):
# Get the directory where the Python script resides
script_dir = os.path.dirname(os.path.abspath(__file__))
# Construct the full path to the config file relative to the script directory
config_path = os.path.join(script_dir, "configs", f"{modelname}.json")
try:
with open(config_path, 'r') as file:
config = json.load(file)
return config
except FileNotFoundError:
log.error(f"Config file {config_path} not found.")
sys.exit(1)

# config.json example:
# {
# "pre": "<PRE> ",
# "middle": " <MIDDLE>",
# "suffix": " <SUFFIX>",
# "eot": " <EOT>"
# }
def fill_in_the_middle(config, prompt):
"""
Searches for the string '<FILL_IN_HERE>' in the prompt and
creates a model specific fill-in-the-middle-prompt.
"""
parts = prompt.split('<FILL_IN_HERE>')
log.debug(parts)

if len(parts) != 2:
log.error("Prompt does not contain '<FILL_IN_HERE>'.")
sys.exit(1)

newprompt = config["pre"] + parts[0] + config["suffix"] + parts[1] + config["middle"]
log.debug(newprompt)

return newprompt

def generate_code_completion(config, prompt, baseurl, model, options):
def generate_code_completion(prompt, baseurl, model, options):
headers = {
'Content-Type': 'application/json',
'Accept': '*/*',
Expand All @@ -64,11 +27,12 @@ def generate_code_completion(config, prompt, baseurl, model, options):
log.debug('endpoint: ' + endpoint)

# generate model specific prompt
prompt = fill_in_the_middle(config, prompt)
prompt, suffix = prompt.split('<FILL_IN_HERE>')

data = {
'model': model,
'prompt': prompt,
'suffix': suffix,
'stream': False,
'raw' : True,
'options': options
Expand All @@ -81,17 +45,9 @@ def generate_code_completion(config, prompt, baseurl, model, options):
json_response = response.json()
log.debug('response: ' + json.dumps(json_response, indent=4))
completion = response.json().get('response')
log.info('completion:' + completion)
log.info('completion: ' + completion)

# find index of sub string
try:
index = completion.find(config.get('eot', '<EOT>'))
if index != -1:
completion = completion[:index] # remove EOT marker
except:
pass

return completion.rstrip()
return completion.strip()
else:
raise Exception(f"Error: {response.status_code} - {response.text}")

Expand All @@ -114,11 +70,6 @@ def generate_code_completion(config, prompt, baseurl, model, options):
except:
options = DEFAULT_OPTIONS

# strip suffix (e.g ':7b-code') from modelname
modelname = args.model
modelname = modelname.rsplit(':', 1)[0]
config = load_config(modelname)

prompt = sys.stdin.read()
response = generate_code_completion(config, prompt, args.url, args.model, options)
response = generate_code_completion(prompt, args.url, args.model, options)
print(response, end='')