""" title: Prompt Quality Evaluation author: Italo Santos date: 2024-10-12 updated: 2024-10-27 version: 1.1 license: MIT description: A function for evaluating prompt quality using NLP techniques, with language detection for pt-br and en-us. requirements: spacy, langdetect """ import os import re from typing import List, Optional from pydantic import BaseModel, Field # Workaround to installs necessary dependencies to handles multiple ephemeral instances. # For more details, visit: https://github.com/open-webui/open-webui/discussions/6484 try: import spacy except ImportError: import subprocess subprocess.run(["pip", "install", "spacy"]) import spacy try: from langdetect import detect except ImportError: import subprocess subprocess.run(["pip", "install", "langdetect"]) from langdetect import detect class Filter: class Valves(BaseModel): # Assign a priority level to the filter pipeline. priority: int = Field( default=5, description="Priority level for the filter operations." ) def __init__(self): self.file_handler = False self.type = "filter" self.name = "Prompt Quality Evaluator with Language Detection" self.valves = self.Valves() # Ensure the necessary spaCy models are downloaded and available self.nlp_models = {} spacy.cli.download("en_core_web_sm") spacy.cli.download("pt_core_news_sm") # Load spaCy models self.nlp_models["en"] = spacy.load("en_core_web_sm") self.nlp_models["pt"] = spacy.load("pt_core_news_sm") def get_last_user_message(self, messages: List[dict]) -> str: """ Retrieves the last message content from the user in a list of messages. Args: messages (List[dict]): A list of message dictionaries, where each dictionary contains keys such as "role" and "content". Returns: str: The content of the last user message. If the content is a list, it returns the text from the first item with type "text". If no user message is found, returns None. """ for message in reversed(messages): if message["role"] == "user": if isinstance(message["content"], list): for item in message["content"]: if item["type"] == "text": return item["text"] return message["content"] return None def detect_language(self, text): try: return "en" if detect(text) else "pt" except Exception as e: return "pt" # Default to Portuguese in case of error def evaluate_prompt(self, prompt: str) -> float: """ Evaluate the prompt and return a score between 0 and 100. """ score = 100 deductions = [] bonuses = [] # Detect language. Default to English lang = self.detect_language(prompt) nlp = self.nlp_models.get(lang, self.nlp_models["en"]) doc = nlp(prompt) # Vague words and named entities in one loop vague_words = { "en": { "something", "things", "stuff", "maybe", "somehow", "etc", "kind of", }, "pt": { "algo", "coisas", "talvez", "de alguma forma", "etc", "mais ou menos", }, } current_vague_words = vague_words.get(lang, vague_words["en"]) num_vague_words = 0 num_entities = 0 for token in doc: if token.text.lower() in current_vague_words: num_vague_words += 1 for ent in doc.ents: num_entities += 1 # 1. Be Clear and Specific if num_vague_words > 0: deduction = num_vague_words * 10 deductions.append(deduction) else: bonuses.append(5) # 2. Provide Context if num_entities < 2: deduction = 15 deductions.append(deduction) else: bonuses.append(10) # 3. Specify the Desired Format format_keywords = { "en": { "list", "explain", "summarize", "generate", "provide", "create", "in bullet points", "as a table", }, "pt": { "listar", "explicar", "resumir", "gerar", "fornecer", "criar", "em tópicos", "como uma tabela", }, } num_format_keywords = sum( 1 for word in format_keywords[lang] if word in prompt.lower() ) if num_format_keywords == 0: deductions.append(10) else: bonus = num_format_keywords * 5 bonuses.append(bonus) # 4. Use Examples if "e.g." in prompt.lower() or "exemplo" in prompt.lower(): bonuses.append(10) else: deductions.append(10) # 5. Ask Direct Questions num_questions = prompt.count("?") if num_questions > 0: bonus = min(num_questions * 5, 15) bonuses.append(bonus) else: deductions.append(10) # 6. Avoid Bias and Leading Language biased_words = { "en": {"always", "never", "worst", "best", "all", "none", "should", "must"}, "pt": { "sempre", "nunca", "pior", "melhor", "todos", "nenhum", "deveria", "deve", }, } num_biased_words = sum( 1 for token in doc if token.text.lower() in biased_words[lang] ) if num_biased_words > 0: deduction = num_biased_words * 10 deductions.append(deduction) # 7. Leverage Instructional Cues if doc[0].pos_ == "VERB" and doc[0].tag_ in ("VB", "VBP", "VBZ"): bonuses.append(10) else: deductions.append(5) # 8. Set the Tone or Style if Necessary tone_keywords = { "en": { "formal", "informal", "professional", "friendly", "simple terms", "as a high school student", "executive", }, "pt": { "formal", "informal", "profissional", "amigável", "termos simples", "como um estudante do ensino médio", "executivo", }, } num_tone_keywords = sum( 1 for phrase in tone_keywords[lang] if phrase in prompt.lower() ) if num_tone_keywords > 0: bonus = num_tone_keywords * 5 bonuses.append(bonus) else: deductions.append(10) # 9. Include Constraints and Requirements if re.search(r"\b\d+\b", prompt): bonuses.append(10) else: deductions.append(10) # 10. Anticipate Potential Misinterpretations ambiguous_pronouns = { "en": {"it", "they", "them", "this", "that", "these", "those"}, "pt": { "isso", "isto", "aquilo", "eles", "elas", "este", "esta", "estes", "estas", }, } num_ambiguous_pronouns = sum( 1 for token in doc if token.text.lower() in ambiguous_pronouns[lang] ) if num_ambiguous_pronouns > 1: deduction = num_ambiguous_pronouns * 5 deductions.append(deduction) # 11. Length and Brevity word_count = len(doc) if word_count < 5: deductions.append(25) # Calculate final score total_deduction = sum(deductions) total_bonus = sum(bonuses) final_score = max(0, min(100, score - total_deduction + total_bonus)) # Scoring Classification if final_score <= 40: classification = "Prompt is unclear, too vague, or grammatically incorrect." elif 40 < final_score <= 70: classification = ( "Prompt has some clarity but lacks specificity or structure." ) elif 70 < final_score <= 90: classification = "Prompt is well-written but may lack some detail." else: classification = "Prompt follows best practices and should generate high-quality outputs." return final_score, classification async def inlet(self, body: dict, __user__: Optional[dict] = None) -> dict: print(f"inlet:{__name__}") self.user_message = self.get_last_user_message(body.get("messages", [])) return body async def outlet( self, body: dict, __user__: Optional[dict] = None, __event_emitter__=None ) -> dict: print(f"outlet:{__name__}") print(f"outlet:body:{body}") # Evaluate the prompt prompt_score, prompt_classification = self.evaluate_prompt(self.user_message) await __event_emitter__( { "type": "status", "data": { "description": f"Prompt Score: {prompt_score} - {prompt_classification}", "done": True, }, } ) return body