What I’m trying to do:
My app has gotten a number of users and I am encountering a number of errors. I am working with Jay and Stu on them and sent an email to Anvil support, where Ryan asked me to post about them in the forums. We have checked each of these extensively and couldnt solve them, so would love support!
Here I have a background funtion that failed on me and I don’t know why.
Error: anvil.server.ExecutionTerminatedError: Server code exited unexpectedly: 11e0cd3edf
Time: Wed 17.4 around 22:00 CEST
Session ID:
XVKCNNCITVBYFJK7XUCQFIXMI3OQGIZH
The background task and all the tasks it calls:
# Function that handles all the AI calls. Gathers the necessary info, creates a task cue and runs up to simultaneous_ai_tasks
# at a time that each generate the cloze deletions for a particular learning card. Continues until all tasks have been
# created. The inidividual tasks handle all the logic of what should happen.
@anvil.server.background_task
def generate_AI_lcs(session_id, generate_cloze, simultaneous_ai_tasks=3, lc_indeces=None, retries=3, max_wait_time=30):
print(f"Entered generate_AI_lcs function with variables session_id: {session_id}, generate_cloze: {generate_cloze}, simultaneous_ai_tasks:{simultaneous_ai_tasks}, lc_indeces:{lc_indeces}, retries={retries}, max_wait_time={max_wait_time}")
# Initialize variables
task_queue = {}
active_tasks = {}
task_id = 0
start_time = time.time()
#TODO: Here rows may have been deleted already. Currently chatching that in get_learning_cards, but could check for it here (Although why?)
learning_cards = get_learning_cards(session_id, lc_indeces=lc_indeces)
# Sort learning_cards by 'pdf_lc_index'
learning_cards = sorted(learning_cards, key=lambda x: x['pdf_lc_index'], reverse=False)
if learning_cards != []:
print(f"Processing {len(learning_cards)} lcs with pdf_lc_index from {learning_cards[0]['pdf_lc_index']} to {learning_cards[-1]['pdf_lc_index']}")
else:
print("No learning cards to process.")
# If we are not to generate any cloze we just set the ai_generated handle to false and return
if not generate_cloze:
print('generate_cloze is False, so just setting all lcs to ai_generated = False and finishing')
for lc_row in learning_cards:
if row_deleted(lc_row): #skips this row if user has deleted the row in the meantime
print(f'Row {lc_row} was deleted.')
continue
lc_row['ai_generated'] = False
return
# Populate the task_queue dictionary, but only while they have credits
available_generation_credits, user = get_available_generation_credits(session_id)
for i, lc in enumerate(learning_cards):
if available_generation_credits is not None and task_id >= available_generation_credits:
session = app_tables.session_info.get_by_id(session_id)
if session['show_generation_limit_reached_msg'] is None: session['show_generation_limit_reached_msg'] = True #only if not already set before
lc['ai_generated'] = False
continue # Stop processing if the available_generation_credits limit is reached
if row_deleted(lc): #skips this row if user has deleted the row in the meantime
print(f'Row {lc} was deleted.')
continue
if lc['lc_type'] == 'pic':
lc['ai_generated'] = False
continue
ai_prompt_text = lc['ai_prompt']
ai_token_num = lc['ai_token_num']
lc_html_text = lc['learning_card']
lc_row_id = lc.get_id() # Getting the Anvil row ID
ai_prompt_text += extract_html_start_sequence(lc['learning_card'])
task_queue[task_id] = {
'ai_prompt_text': ai_prompt_text,
'ai_token_num': ai_token_num,
'lc_html_text': lc_html_text,
'lc_row_id': lc_row_id # Storing the Anvil row ID
}
task_id += 1
#print(f"before: user: {user['email']}, available generation credits: {user['available_generation_credits']}, task_id: {task_id}")
if user: user['available_generation_credits'] -= task_id #case when a not logged in person generates cards requires the if statement
#print(f"after: user: {user['email']}, available generation credits: {user['available_generation_credits']}")
print(f"Task_queue length={len(task_queue)}")
# Launches the first x=simultaneous_ai_tasks tasks
# Sort the task_queue in reverse order first so we start by popping the lowest pdf_lc_index first
sorted_task_queue_list = sorted(task_queue.items(), key=lambda item: item[0], reverse=True)
task_queue = dict(sorted_task_queue_list)
for i in range(min(simultaneous_ai_tasks, len(task_queue))):
new_task_id, new_task_info = task_queue.popitem()
active_tasks[new_task_id] = new_task_info
launch_AI_task(new_task_id, active_tasks, session_id, retries, max_wait_time)
print(f"Active_task length={len(active_tasks)}, task_queue length={len(task_queue)}")
# Monitors the active tasks and as individual ones complete launches a new task,
# up to x=simultaneous_ai_tasks tasks until all tasks have been added
while active_tasks or task_queue:
manage_active_tasks(active_tasks, task_queue, session_id, retries, max_wait_time)
print('generate_AI_lcs completed')
# Function for launching AI tasks
def launch_AI_task(task_id, active_tasks, session_id, retries, max_wait_time):
active_task = active_tasks[task_id]
ai_prompt_text = active_task['ai_prompt_text']
ai_token_num = active_task['ai_token_num']
lc_html_text = active_task['lc_html_text']
lc_row_id = active_task['lc_row_id']
print(f'launching AI task for: {lc_html_text} with lc_row_id: {lc_row_id}')
active_task['task'] = anvil.server.launch_background_task('generate_AI_lc', session_id, ai_prompt_text, ai_token_num, lc_row_id, lc_html_text, retries, max_wait_time)
# Function to manage the active tasks
def manage_active_tasks(active_tasks, task_queue, session_id, retries, max_wait_time):
print(f'Entered manage_active_tasks with {len(active_tasks)} active tasks and a task_queue of {len(task_queue)} elements.')
for task_id, task_info in list(active_tasks.items()):
task = task_info['task']
#If task is finished or died we kill it and run next task. What to do on fail we handle in the task. While running status remains none.
if task.get_termination_status() is not None:
print(f'An ai task completed.')
active_tasks.pop(task_id)
if task_queue:
print(f'Launching another.')
new_task_id, new_task_info = task_queue.popitem()
active_tasks[new_task_id] = new_task_info
launch_AI_task(new_task_id, active_tasks, session_id, retries, max_wait_time)
time.sleep(5)
# Individual function that is run for each lc for which we want to generate a cloze deletion.
# It runs the openai call, waits some time for a response and retries a number of times.
# If it gets a response it then analyes the response and updates the learning card row.
# If it doesnt get a response or the response is not clean it ends and the lc row remains without cloze deletion.s
@anvil.server.background_task
def generate_AI_lc(session_id, ai_prompt_text, ai_token_num, lc_row_id, lc_html_text, retries, max_wait_time):
# Get the lc row
lc_row = app_tables.learning_cards.get_by_id(lc_row_id)
if row_deleted(lc_row): #stop the task if user has deleted the row in the meantime
print(f'Row {lc_row} was deleted.')
return
# Step 1: Make OpenAI Request
print(f'Making the openai call')
openai_message_content = retry_openai_request(ai_prompt_text, ai_token_num, retries, max_wait_time)
print(f'Openai call finished')
# 1a: Check for failure
if openai_message_content == "failed":
print(f"OpenAI request failed after {retries} retries.")
if row_deleted(lc_row): #stop the task if user has deleted the row in the meantime
print(f'Row {lc_row} was deleted.')
return
lc_row['ai_generated'] = False
print(f'returning credit')
return_generation_credit(session_id)
return
# Step 2: Process OpenAI Response
print(f'openai_message_content is: {openai_message_content}')
openAI_html = openai_message_content
cloze_checker = class_clean_openAI_output.ClozeModifier()
cleaned_lc = cloze_checker.modify_text(lc_html_text, openAI_html)
# 2a: If openAI Response was not a valid cloze card
if cleaned_lc == "Error":
print(f"Error in cleaning the learning card. Going with html_lc and no cloze.")
if row_deleted(lc_row): #stop the task if user has deleted the row in the meantime
print(f'Row {lc_row} was deleted.')
return
lc_row['ai_generated'] = False
print(f'returning credit')
return_generation_credit(session_id)
return
# Step 3: Update Learning Card
print(f'Updated row with AI lc')
if row_deleted(lc_row): #stop the task if user has deleted the row in the meantime
print(f'Row {lc_row} was deleted.')
return
lc_row['learning_card'] = cleaned_lc
lc_row['ai_generated'] = True
print(f"lc with pdf_lc_index {lc_row['pdf_lc_index']} updated.")
# Helper function that generates the openAI call and waits for a response, retrying a number of times.
def retry_openai_request(ai_prompt, ai_token_num, retries, max_wait_time):
wait_time = 5 # in seconds
num_checks = max_wait_time // wait_time
for i in range(retries):
# Launch the OpenAI API background task
print(f'launching openai background task, try {i+1}')
task = anvil.server.launch_background_task('openai_background_task', ai_prompt, ai_token_num)
for j in range(num_checks):
# Check the task status
if task.get_termination_status() is not None:
# If the task is complete, get the result.
if task.get_termination_status() == "completed":
print('task succeeded')
return task.get_return_value()
elif task.get_termination_status() == "failed":
# If the task failed, handle the exception here or log it
try:
# This will re-throw the exception
task.get_error()
except Exception as e:
print(f'Task failed with exception: {e}')
break # Exit the inner loop and retry the task
time.sleep(wait_time) # Wait for 5 seconds before checking again
# If we reached here, the task has either failed or didn't complete in time.
# Cancel the existing task and retry.
print(f'Task failed {i+1} times. Killing it.')
task.kill()
return "failed" # Return 'failed' after 3 retries
# The openAI call
@anvil.server.background_task
def openai_background_task(ai_prompt, ai_token_num):
start_time = time.time()
token_multiplyer = 1.5 # how much longer the answer can be than the input
openAI_key = anvil.secrets.get_secret("openAI_key")
client = OpenAI(api_key=openAI_key)
print(f"ai_token_num: {ai_token_num}, token_multiplier: {token_multiplyer}")
response = client.chat.completions.create(
model="gpt-3.5-turbo-1106",
messages=[
{
"role": "user",
"content": ai_prompt
}
],
temperature=0.2,
max_tokens=int(ai_token_num * token_multiplyer),
top_p=1,
frequency_penalty=0,
presence_penalty=0
)
time_taken = time.time()-start_time
openai_message_cotent = response.choices[0].message.content
print(f'Completing API call took {time_taken} seconds')
print(f"openai_message_cotent: {openai_message_cotent}")
return openai_message_cotent
# Helper function that gets the starting html of the learning card text to add to the end of the prompt
def extract_html_start_sequence(html_lc):
html_start_sequence = ""
# Loop while the remaining_prompt starts with an HTML tag
while html_lc.startswith("<"):
# Find the first HTML tag in the remaining_prompt
tag_match = re.match(r'<[^>]+>', html_lc)
if tag_match:
# Extract the HTML tag and add it to start_html_tags
html_tag = tag_match.group()
html_start_sequence += html_tag
# Remove the found HTML tag from the remaining_prompt
html_lc = html_lc[len(html_tag):].strip()
else:
# If no valid HTML tag is found, break the loop
break
return html_start_sequence
def row_deleted(row):
if row is None:
return True
try:
test = row['lc_index']
except anvil.tables.RowDeleted:
print(f'Row {row} was deleted.')
return True
return False
def get_available_generation_credits(session_id):
session = app_tables.session_info.get_by_id(session_id)
user = session['user']
if user:
available_generation_credits = user['available_generation_credits']
else:
#dealt with no user case by restricting the number of rows passed to 50 in bckgrnd_generate_lcs
available_generation_credits = None
print (f"available_genartion_credits: {available_generation_credits}, user_email: {user['email'] if user else 'no user'}")
return available_generation_credits, user
def return_generation_credit(session_id):
session = app_tables.session_info.get_by_id(session_id)
user = session['user']
if user:
print(f"available generation credits before return: {user['available_generation_credits']}")
user['available_generation_credits'] +=1
print(f"available generation credits after return: {user['available_generation_credits']}")