Resolves #1: Multi-Dimensional Quality Scoring

scorer.py

@@ -0,0 +1,165 @@
+import json
+import re
+import ast
+
+class QualityScorer:
+    def __init__(self):
+        self.weights = {
+            "completeness": 0.30,
+            "format_compliance": 0.20,
+            "coverage": 0.25,
+            "clarity": 0.15,
+            "validity": 0.10
+        }
+
+    def detect_format(self, submission: str) -> str:
+        submission = submission.strip()
+        if submission.startswith('{') and submission.endswith('}'):
+            try:
+                json.loads(submission)
+                return "json"
+            except:
+                pass
+
+        if submission.startswith('[') and submission.endswith(']'):
+            try:
+                json.loads(submission)
+                return "json"
+            except:
+                pass
+
+        if "```" in submission or submission.startswith('# ') or '**' in submission:
+            return "markdown"
+
+        try:
+            ast.parse(submission)
+            if "def " in submission or "class " in submission or "import " in submission:
+                return "code"
+        except:
+            pass
+
+        return "text"
+
+    def _score_completeness(self, submission: str, fmt: str, rubric: dict) -> tuple[float, str]:
+        score = 1.0
+        feedback = "Submission is fully complete."
+        if fmt == "json":
+            try:
+                data = json.loads(submission)
+                if isinstance(data, dict):
+                    expected_keys = rubric.get("expected_keys", [])
+                    missing = [k for k in expected_keys if k not in data]
+                    if missing:
+                        score = max(0.0, 1.0 - (len(missing) / len(expected_keys)))
+                        feedback = f"Missing required keys: {', '.join(missing)}."
+            except:
+                score = 0.0
+                feedback = "Failed to parse JSON for completeness check."
+        elif fmt == "markdown":
+            expected_sections = rubric.get("expected_sections", [])
+            missing = [s for s in expected_sections if s.lower() not in submission.lower()]
+            if missing:
+                score = max(0.0, 1.0 - (len(missing) / len(expected_sections)))
+                feedback = f"Missing sections: {', '.join(missing)}."
+        return score, feedback
+
+    def _score_format_compliance(self, submission: str, fmt: str) -> tuple[float, str]:
+        if fmt == "json":
+            try:
+                json.loads(submission)
+                return 1.0, "Perfect JSON format compliance."
+            except:
+                return 0.0, "Invalid JSON format."
+        elif fmt == "markdown":
+            if "```" in submission or "#" in submission:
+                return 1.0, "Good use of markdown formatting."
+            return 0.5, "Missing typical markdown elements."
+        elif fmt == "code":
+            try:
+                ast.parse(submission)
+                return 1.0, "Code is syntactically valid."
+            except:
+                return 0.0, "Syntax errors in code."
+        else:
+            if len(submission.strip()) > 0:
+                return 1.0, "Valid plain text."
+            return 0.0, "Empty text submission."
+
+    def _score_coverage(self, submission: str, rubric: dict) -> tuple[float, str]:
+        keywords = rubric.get("keywords", [])
+        if not keywords:
+            return 1.0, "No specific keywords required for coverage."
+
+        text = submission.lower()
+        found = [k for k in keywords if k.lower() in text]
+        score = len(found) / len(keywords)
+        if score == 1.0:
+            return 1.0, "Excellent coverage of all required keywords."
+        elif score > 0.5:
+            return score, "Good coverage, but some keywords are missing."
+        else:
+            return score, "Poor coverage of required topics."
+
+    def _score_clarity(self, submission: str) -> tuple[float, str]:
+        words = submission.split()
+        if len(words) == 0:
+            return 0.0, "Empty submission lacks clarity."
+        avg_word_len = sum(len(w) for w in words) / len(words)
+        if 4 <= avg_word_len <= 8:
+            return 1.0, "Clear and readable vocabulary."
+        else:
+            return 0.7, "Vocabulary might be too simple or overly complex."
+
+    def _score_validity(self, submission: str) -> tuple[float, str]:
+        if "TODO" in submission or "FIXME" in submission:
+            return 0.5, "Contains unresolved TODOs or FIXMEs."
+        return 1.0, "Content appears logically valid and complete."
+
+    def score(self, submission: str, rubric: dict = None) -> dict:
+        if rubric is None:
+            rubric = {}
+
+        fmt = self.detect_format(submission)
+
+        comp_score, comp_fb = self._score_completeness(submission, fmt, rubric)
+        fmt_score, fmt_fb = self._score_format_compliance(submission, fmt)
+        cov_score, cov_fb = self._score_coverage(submission, rubric)
+        clar_score, clar_fb = self._score_clarity(submission)
+        val_score, val_fb = self._score_validity(submission)
+
+        scores = {
+            "completeness": comp_score,
+            "format_compliance": fmt_score,
+            "coverage": cov_score,
+            "clarity": clar_score,
+            "validity": val_score
+        }
+
+        feedback = [
+            f"Completeness: {comp_fb}",
+            f"Format Compliance: {fmt_fb}",
+            f"Coverage: {cov_fb}",
+            f"Clarity: {clar_fb}",
+            f"Validity: {val_fb}"
+        ]
+
+        weighted_score = sum(scores[k] * self.weights[k] for k in scores)
+
+        if weighted_score >= 0.9:
+            rating = "A"
+        elif weighted_score >= 0.8:
+            rating = "B"
+        elif weighted_score >= 0.7:
+            rating = "C"
+        elif weighted_score >= 0.6:
+            rating = "D"
+        else:
+            rating = "F"
+
+        return {
+            "weighted_score": round(weighted_score, 4),
+            "quality_rating": rating,
+            "scores": scores,
+            "feedback": feedback,
+            "pass_threshold": weighted_score >= 0.70
+        }

test_scorer.py

@@ -0,0 +1,59 @@
+import time
+from scorer import QualityScorer
+
+def test_scoring_performance_and_accuracy():
+    scorer = QualityScorer()
+    rubric = {
+        "expected_keys": ["title", "content", "author"],
+        "expected_sections": ["Introduction", "Methodology"],
+        "keywords": ["AI", "scoring", "algorithm"]
+    }
+
+    submissions = [
+        # JSON
+        ('{"title": "Test", "content": "AI scoring algorithm is good.", "author": "John"}', 1.0),
+        ('{"title": "Test", "content": "AI is good."}', 0.7),
+
+        # Markdown
+        ('# Introduction\nAI is here.\n# Methodology\nScoring algorithm.', 1.0),
+        ('# Introduction\nJust some text without keywords.', 0.6),
+
+        # Code
+        ('def evaluate_ai():\n    return "scoring algorithm"', 0.8),
+
+        # Text
+        ('This is a plain text submission about AI scoring algorithm.', 0.9),
+    ] * 20 # 120 submissions
+
+    start_time = time.time()
+
+    for sub, expected in submissions:
+        result = scorer.score(sub, rubric)
+        assert result is not None
+        assert 0.0 <= result["weighted_score"] <= 1.0
+        assert len(result["scores"]) == 5
+
+    duration = time.time() - start_time
+    print(f"Processed {len(submissions)} submissions in {duration:.4f} seconds")
+    assert duration < 10.0, "Performance failed!"
+
+    # Generate sample scorecard
+    sample_result = scorer.score(submissions[0][0], rubric)
+    print("\nSample Scorecard:")
+    print("=" * 40)
+    for k, v in sample_result.items():
+        if isinstance(v, list):
+            print(f"{k}:")
+            for item in v:
+                print(f"  - {item}")
+        elif isinstance(v, dict):
+            print(f"{k}:")
+            for sk, sv in v.items():
+                print(f"  - {sk}: {sv}")
+        else:
+            print(f"{k}: {v}")
+    print("=" * 40)
+
+if __name__ == "__main__":
+    test_scoring_performance_and_accuracy()
+    print("All tests passed.")