diff --git a/scorer.py b/scorer.py new file mode 100644 index 0000000..233c344 --- /dev/null +++ b/scorer.py @@ -0,0 +1,165 @@ +import json +import re +import ast + +class QualityScorer: + def __init__(self): + self.weights = { + "completeness": 0.30, + "format_compliance": 0.20, + "coverage": 0.25, + "clarity": 0.15, + "validity": 0.10 + } + + def detect_format(self, submission: str) -> str: + submission = submission.strip() + if submission.startswith('{') and submission.endswith('}'): + try: + json.loads(submission) + return "json" + except: + pass + + if submission.startswith('[') and submission.endswith(']'): + try: + json.loads(submission) + return "json" + except: + pass + + if "```" in submission or submission.startswith('# ') or '**' in submission: + return "markdown" + + try: + ast.parse(submission) + if "def " in submission or "class " in submission or "import " in submission: + return "code" + except: + pass + + return "text" + + def _score_completeness(self, submission: str, fmt: str, rubric: dict) -> tuple[float, str]: + score = 1.0 + feedback = "Submission is fully complete." + if fmt == "json": + try: + data = json.loads(submission) + if isinstance(data, dict): + expected_keys = rubric.get("expected_keys", []) + missing = [k for k in expected_keys if k not in data] + if missing: + score = max(0.0, 1.0 - (len(missing) / len(expected_keys))) + feedback = f"Missing required keys: {', '.join(missing)}." + except: + score = 0.0 + feedback = "Failed to parse JSON for completeness check." + elif fmt == "markdown": + expected_sections = rubric.get("expected_sections", []) + missing = [s for s in expected_sections if s.lower() not in submission.lower()] + if missing: + score = max(0.0, 1.0 - (len(missing) / len(expected_sections))) + feedback = f"Missing sections: {', '.join(missing)}." + return score, feedback + + def _score_format_compliance(self, submission: str, fmt: str) -> tuple[float, str]: + if fmt == "json": + try: + json.loads(submission) + return 1.0, "Perfect JSON format compliance." + except: + return 0.0, "Invalid JSON format." + elif fmt == "markdown": + if "```" in submission or "#" in submission: + return 1.0, "Good use of markdown formatting." + return 0.5, "Missing typical markdown elements." + elif fmt == "code": + try: + ast.parse(submission) + return 1.0, "Code is syntactically valid." + except: + return 0.0, "Syntax errors in code." + else: + if len(submission.strip()) > 0: + return 1.0, "Valid plain text." + return 0.0, "Empty text submission." + + def _score_coverage(self, submission: str, rubric: dict) -> tuple[float, str]: + keywords = rubric.get("keywords", []) + if not keywords: + return 1.0, "No specific keywords required for coverage." + + text = submission.lower() + found = [k for k in keywords if k.lower() in text] + score = len(found) / len(keywords) + if score == 1.0: + return 1.0, "Excellent coverage of all required keywords." + elif score > 0.5: + return score, "Good coverage, but some keywords are missing." + else: + return score, "Poor coverage of required topics." + + def _score_clarity(self, submission: str) -> tuple[float, str]: + words = submission.split() + if len(words) == 0: + return 0.0, "Empty submission lacks clarity." + avg_word_len = sum(len(w) for w in words) / len(words) + if 4 <= avg_word_len <= 8: + return 1.0, "Clear and readable vocabulary." + else: + return 0.7, "Vocabulary might be too simple or overly complex." + + def _score_validity(self, submission: str) -> tuple[float, str]: + if "TODO" in submission or "FIXME" in submission: + return 0.5, "Contains unresolved TODOs or FIXMEs." + return 1.0, "Content appears logically valid and complete." + + def score(self, submission: str, rubric: dict = None) -> dict: + if rubric is None: + rubric = {} + + fmt = self.detect_format(submission) + + comp_score, comp_fb = self._score_completeness(submission, fmt, rubric) + fmt_score, fmt_fb = self._score_format_compliance(submission, fmt) + cov_score, cov_fb = self._score_coverage(submission, rubric) + clar_score, clar_fb = self._score_clarity(submission) + val_score, val_fb = self._score_validity(submission) + + scores = { + "completeness": comp_score, + "format_compliance": fmt_score, + "coverage": cov_score, + "clarity": clar_score, + "validity": val_score + } + + feedback = [ + f"Completeness: {comp_fb}", + f"Format Compliance: {fmt_fb}", + f"Coverage: {cov_fb}", + f"Clarity: {clar_fb}", + f"Validity: {val_fb}" + ] + + weighted_score = sum(scores[k] * self.weights[k] for k in scores) + + if weighted_score >= 0.9: + rating = "A" + elif weighted_score >= 0.8: + rating = "B" + elif weighted_score >= 0.7: + rating = "C" + elif weighted_score >= 0.6: + rating = "D" + else: + rating = "F" + + return { + "weighted_score": round(weighted_score, 4), + "quality_rating": rating, + "scores": scores, + "feedback": feedback, + "pass_threshold": weighted_score >= 0.70 + } diff --git a/test_scorer.py b/test_scorer.py new file mode 100644 index 0000000..7594fb5 --- /dev/null +++ b/test_scorer.py @@ -0,0 +1,59 @@ +import time +from scorer import QualityScorer + +def test_scoring_performance_and_accuracy(): + scorer = QualityScorer() + rubric = { + "expected_keys": ["title", "content", "author"], + "expected_sections": ["Introduction", "Methodology"], + "keywords": ["AI", "scoring", "algorithm"] + } + + submissions = [ + # JSON + ('{"title": "Test", "content": "AI scoring algorithm is good.", "author": "John"}', 1.0), + ('{"title": "Test", "content": "AI is good."}', 0.7), + + # Markdown + ('# Introduction\nAI is here.\n# Methodology\nScoring algorithm.', 1.0), + ('# Introduction\nJust some text without keywords.', 0.6), + + # Code + ('def evaluate_ai():\n return "scoring algorithm"', 0.8), + + # Text + ('This is a plain text submission about AI scoring algorithm.', 0.9), + ] * 20 # 120 submissions + + start_time = time.time() + + for sub, expected in submissions: + result = scorer.score(sub, rubric) + assert result is not None + assert 0.0 <= result["weighted_score"] <= 1.0 + assert len(result["scores"]) == 5 + + duration = time.time() - start_time + print(f"Processed {len(submissions)} submissions in {duration:.4f} seconds") + assert duration < 10.0, "Performance failed!" + + # Generate sample scorecard + sample_result = scorer.score(submissions[0][0], rubric) + print("\nSample Scorecard:") + print("=" * 40) + for k, v in sample_result.items(): + if isinstance(v, list): + print(f"{k}:") + for item in v: + print(f" - {item}") + elif isinstance(v, dict): + print(f"{k}:") + for sk, sv in v.items(): + print(f" - {sk}: {sv}") + else: + print(f"{k}: {v}") + print("=" * 40) + +if __name__ == "__main__": + test_scoring_performance_and_accuracy() + print("All tests passed.")