|
|
@@ -0,0 +1,315 @@
|
|
|
+---
|
|
|
+phase: 02-assembly-integration
|
|
|
+plan: 01
|
|
|
+type: execute
|
|
|
+wave: 1
|
|
|
+depends_on: []
|
|
|
+files_modified:
|
|
|
+ - database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php
|
|
|
+ - app/Jobs/AssembleExamTaskJob.php
|
|
|
+ - app/Services/LearningAnalyticsService.php
|
|
|
+autonomous: true
|
|
|
+requirements:
|
|
|
+ - ASM-01
|
|
|
+ - ASM-02
|
|
|
+ - ASM-03
|
|
|
+
|
|
|
+must_haves:
|
|
|
+ truths:
|
|
|
+ - "Every question that passes through AssembleExamTaskJob::hydrateQuestions() has calibrated difficulty applied when available in question_difficulty_calibrations"
|
|
|
+ - "Every question that passes through LearningAnalyticsService::generateIntelligentExam() has calibrated difficulty applied before distribution/sorting, regardless of which code path was taken"
|
|
|
+ - "Questions without a row in question_difficulty_calibrations retain their original questions.difficulty value unchanged"
|
|
|
+ - "Difficulty distribution strategy is active by default even when ExamTypeStrategy::buildParams() is not called or throws"
|
|
|
+ artifacts:
|
|
|
+ - path: "database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php"
|
|
|
+ provides: "Adds difficulty_source column to paper_questions table"
|
|
|
+ contains: "difficulty_source"
|
|
|
+ - path: "app/Jobs/AssembleExamTaskJob.php"
|
|
|
+ provides: "Calls QuestionDifficultyResolver::applyCalibratedDifficulty() in hydrateQuestions()"
|
|
|
+ contains: "applyCalibratedDifficulty"
|
|
|
+ - path: "app/Services/LearningAnalyticsService.php"
|
|
|
+ provides: "Applies calibrated difficulty after gathering questions and defaults distribution to true"
|
|
|
+ contains: "applyCalibratedDifficulty"
|
|
|
+ key_links:
|
|
|
+ - from: "app/Jobs/AssembleExamTaskJob.php"
|
|
|
+ to: "app/Services/QuestionDifficultyResolver.php"
|
|
|
+ via: "resolve from container and call applyCalibratedDifficulty()"
|
|
|
+ pattern: "QuestionDifficultyResolver.*applyCalibratedDifficulty"
|
|
|
+ - from: "app/Services/LearningAnalyticsService.php"
|
|
|
+ to: "app/Services/QuestionDifficultyResolver.php"
|
|
|
+ via: "resolve from container and call applyCalibratedDifficulty()"
|
|
|
+ pattern: "QuestionDifficultyResolver.*applyCalibratedDifficulty"
|
|
|
+---
|
|
|
+
|
|
|
+<objective>
|
|
|
+Wire the QuestionDifficultyResolver into the two remaining assembly paths (AssembleExamTaskJob and LearningAnalyticsService) so that all exam assembly uses calibrated difficulty when available, with automatic fallback to original difficulty. Also activate difficulty distribution by default and prepare the paper_questions table for audit tracing.
|
|
|
+
|
|
|
+Purpose: Without this wiring, exams assembled via the async job path (AssembleExamTaskJob) and certain LearningAnalyticsService paths use original (potentially inaccurate) difficulty values, while other paths use calibrated values. This inconsistency means some students get poorly-calibrated exams.
|
|
|
+Output: Migration adding difficulty_source column, resolver calls in AssembleExamTaskJob and LearningAnalyticsService, distribution default set to true.
|
|
|
+</objective>
|
|
|
+
|
|
|
+<execution_context>
|
|
|
+@$HOME/.claude/get-shit-done/workflows/execute-plan.md
|
|
|
+@$HOME/.claude/get-shit-done/templates/summary.md
|
|
|
+</execution_context>
|
|
|
+
|
|
|
+<context>
|
|
|
+@.planning/PROJECT.md
|
|
|
+@.planning/ROADMAP.md
|
|
|
+@.planning/STATE.md
|
|
|
+@.planning/phases/02-assembly-integration/02-RESEARCH.md
|
|
|
+
|
|
|
+<interfaces>
|
|
|
+<!-- Key interfaces the executor needs. Extracted from codebase. -->
|
|
|
+
|
|
|
+From app/Services/QuestionDifficultyResolver.php:
|
|
|
+```php
|
|
|
+class QuestionDifficultyResolver
|
|
|
+{
|
|
|
+ // Returns questions with difficulty overridden and difficulty_source='calibrated' set
|
|
|
+ // for questions found in question_difficulty_calibrations.
|
|
|
+ // Questions NOT in the calibrations table are returned unchanged (original difficulty preserved).
|
|
|
+ public function applyCalibratedDifficulty(array $questions): array
|
|
|
+
|
|
|
+ // Returns map of question_bank_id => calibrated_difficulty
|
|
|
+ public function mapCalibratedDifficulty(array $questionIds): array
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+From app/Jobs/AssembleExamTaskJob.php line 376-398:
|
|
|
+```php
|
|
|
+private function hydrateQuestions(array $questions, array $kpCodes): array
|
|
|
+{
|
|
|
+ $normalized = [];
|
|
|
+ foreach ($questions as $question) {
|
|
|
+ // ... normalization ...
|
|
|
+ $normalized[] = [
|
|
|
+ 'id' => ...,
|
|
|
+ 'difficulty' => isset($question['difficulty']) ? (float) $question['difficulty'] : 0.5,
|
|
|
+ // ...
|
|
|
+ ];
|
|
|
+ }
|
|
|
+ return array_values(array_filter($normalized, fn ($q) => ! empty($q['id'])));
|
|
|
+ // NOTE: No call to questionDifficultyResolver here -- THIS IS THE GAP
|
|
|
+}
|
|
|
+```
|
|
|
+
|
|
|
+From app/Http/Controllers/Api/IntelligentExamController.php line 733 (WORKING REFERENCE):
|
|
|
+```php
|
|
|
+// This is how the controller does it -- replicate this pattern in the job
|
|
|
+return $this->questionDifficultyResolver->applyCalibratedDifficulty($normalized);
|
|
|
+```
|
|
|
+
|
|
|
+From app/Services/LearningAnalyticsService.php line 1554:
|
|
|
+```php
|
|
|
+$enableDistribution = $params['enable_difficulty_distribution'] ?? false;
|
|
|
+// CHANGE TO: $params['enable_difficulty_distribution'] ?? true;
|
|
|
+```
|
|
|
+
|
|
|
+From database/migrations/2025_11_24_000000_create_base_tables_for_testing.php:
|
|
|
+```php
|
|
|
+// paper_questions base schema -- has columns: id, question_text, knowledge_point, timestamps
|
|
|
+// Later migrations add: question_id, question_bank_id, question_type, difficulty, score, etc.
|
|
|
+```
|
|
|
+</interfaces>
|
|
|
+</context>
|
|
|
+
|
|
|
+<tasks>
|
|
|
+
|
|
|
+<task type="auto">
|
|
|
+ <name>Task 1: Add difficulty_source migration and wire resolver into AssembleExamTaskJob</name>
|
|
|
+ <files>database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php, app/Jobs/AssembleExamTaskJob.php</files>
|
|
|
+ <read_first>
|
|
|
+ app/Jobs/AssembleExamTaskJob.php
|
|
|
+ app/Services/QuestionDifficultyResolver.php
|
|
|
+ app/Http/Controllers/Api/IntelligentExamController.php
|
|
|
+ database/migrations/2025_11_23_090143_add_question_type_to_paper_questions_table.php
|
|
|
+ </read_first>
|
|
|
+ <acceptance_criteria>
|
|
|
+ - Migration file exists at `database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php`
|
|
|
+ - `grep -c 'difficulty_source' database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php` returns >= 2
|
|
|
+ - `grep -c 'applyCalibratedDifficulty' app/Jobs/AssembleExamTaskJob.php` returns >= 1
|
|
|
+ - `grep -c 'QuestionDifficultyResolver' app/Jobs/AssembleExamTaskJob.php` returns >= 1
|
|
|
+ - `php artisan migrate --path=database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php` exits 0
|
|
|
+ - AssembleExamTaskJob::hydrateQuestions() return statement passes through applyCalibratedDifficulty()
|
|
|
+ </acceptance_criteria>
|
|
|
+ <action>
|
|
|
+**Step A: Create migration for difficulty_source column.**
|
|
|
+
|
|
|
+Create `database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php` following the pattern from `2025_11_23_090143_add_question_type_to_paper_questions_table.php`:
|
|
|
+
|
|
|
+```php
|
|
|
+<?php
|
|
|
+
|
|
|
+use Illuminate\Database\Migrations\Migration;
|
|
|
+use Illuminate\Database\Schema\Blueprint;
|
|
|
+use Illuminate\Support\Facades\Schema;
|
|
|
+
|
|
|
+return new class extends Migration
|
|
|
+{
|
|
|
+ public function up(): void
|
|
|
+ {
|
|
|
+ if (Schema::hasTable('paper_questions')) {
|
|
|
+ Schema::table('paper_questions', function (Blueprint $table) {
|
|
|
+ $table->string('difficulty_source', 20)->nullable()->default('original')->after('difficulty');
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public function down(): void
|
|
|
+ {
|
|
|
+ if (Schema::hasTable('paper_questions')) {
|
|
|
+ Schema::table('paper_questions', function (Blueprint $table) {
|
|
|
+ $table->dropColumn('difficulty_source');
|
|
|
+ });
|
|
|
+ }
|
|
|
+ }
|
|
|
+};
|
|
|
+```
|
|
|
+
|
|
|
+The column: string(20), nullable, default 'original', placed after the `difficulty` column. Use the `if (Schema::hasTable(...))` guard pattern from the existing migrations. Run the migration with `php artisan migrate`.
|
|
|
+
|
|
|
+**Step B: Wire QuestionDifficultyResolver into AssembleExamTaskJob::hydrateQuestions().**
|
|
|
+
|
|
|
+Open `app/Jobs/AssembleExamTaskJob.php`. Make TWO changes:
|
|
|
+
|
|
|
+1. Add import at the top of the file (after existing imports, around line 6):
|
|
|
+ ```php
|
|
|
+ use App\Services\QuestionDifficultyResolver;
|
|
|
+ ```
|
|
|
+
|
|
|
+2. In the private method `hydrateQuestions()` (line 376-398), modify the return statement. Currently the method returns:
|
|
|
+ ```php
|
|
|
+ return array_values(array_filter($normalized, fn ($q) => ! empty($q['id'])));
|
|
|
+ ```
|
|
|
+ Change it to resolve the resolver from the container and apply calibrated difficulty:
|
|
|
+ ```php
|
|
|
+ $result = array_values(array_filter($normalized, fn ($q) => ! empty($q['id'])));
|
|
|
+
|
|
|
+ return app(QuestionDifficultyResolver::class)->applyCalibratedDifficulty($result);
|
|
|
+ ```
|
|
|
+
|
|
|
+ Use `app(QuestionDifficultyResolver::class)` to resolve from the container rather than constructor injection -- this follows the research recommendation (Pattern B) and avoids changing the job's constructor signature which could affect queue serialization.
|
|
|
+
|
|
|
+IMPORTANT: Do NOT inject via constructor. The job is dispatched to a queue and constructor parameters must be serializable. Use `app()` container resolution inside the method instead. This is the same pattern used elsewhere in the codebase for optional service resolution.
|
|
|
+ </action>
|
|
|
+ <verify>
|
|
|
+ <automated>cd /Volumes/T9/code/math_cms && php artisan migrate --path=database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php 2>&1 && echo "MIGRATION OK" && grep -c 'applyCalibratedDifficulty' app/Jobs/AssembleExamTaskJob.php && grep -c 'QuestionDifficultyResolver' app/Jobs/AssembleExamTaskJob.php</automated>
|
|
|
+ </verify>
|
|
|
+ <done>
|
|
|
+ - Migration adds difficulty_source column (string 20, nullable, default 'original') to paper_questions table
|
|
|
+ - Migration runs successfully without errors
|
|
|
+ - AssembleExamTaskJob::hydrateQuestions() calls applyCalibratedDifficulty() via app() container resolution
|
|
|
+ - QuestionDifficultyResolver import added to AssembleExamTaskJob.php
|
|
|
+ - Questions from mistake-based assembly paths (assemble_type 5, 15) now get calibrated difficulty
|
|
|
+ </done>
|
|
|
+</task>
|
|
|
+
|
|
|
+<task type="auto">
|
|
|
+ <name>Task 2: Wire resolver into LearningAnalyticsService and activate distribution by default</name>
|
|
|
+ <files>app/Services/LearningAnalyticsService.php</files>
|
|
|
+ <read_first>
|
|
|
+ app/Services/LearningAnalyticsService.php
|
|
|
+ app/Services/QuestionDifficultyResolver.php
|
|
|
+ </read_first>
|
|
|
+ <acceptance_criteria>
|
|
|
+ - `grep -c 'applyCalibratedDifficulty' app/Services/LearningAnalyticsService.php` returns >= 1
|
|
|
+ - `grep -c 'QuestionDifficultyResolver' app/Services/LearningAnalyticsService.php` returns >= 1
|
|
|
+ - Line 1554 now reads `$params['enable_difficulty_distribution'] ?? true` (not `?? false`)
|
|
|
+ - Resolver is called on the question pool AFTER gathering from DB and BEFORE selectQuestionsByMastery/distribution
|
|
|
+ </acceptance_criteria>
|
|
|
+ <action>
|
|
|
+Open `app/Services/LearningAnalyticsService.php`. Make THREE changes:
|
|
|
+
|
|
|
+**Change 1: Add import.**
|
|
|
+Add at the top (after existing use statements):
|
|
|
+```php
|
|
|
+use App\Services\QuestionDifficultyResolver;
|
|
|
+```
|
|
|
+
|
|
|
+**Change 2: Change difficulty distribution default to true.**
|
|
|
+Find line 1554 (inside `generateIntelligentExam()` method):
|
|
|
+```php
|
|
|
+$enableDistribution = $params['enable_difficulty_distribution'] ?? false;
|
|
|
+```
|
|
|
+Change to:
|
|
|
+```php
|
|
|
+$enableDistribution = $params['enable_difficulty_distribution'] ?? true;
|
|
|
+```
|
|
|
+
|
|
|
+**Change 3: Apply calibrated difficulty to the full question pool.**
|
|
|
+Find the location in `generateIntelligentExam()` where all questions have been gathered from the bank but BEFORE they are passed to `selectQuestionsByMastery()`. This is around the point where `$allQuestions` or similar variable holds the full candidate pool from `getQuestionsFromBank()` and any supplement questions from `fetchQuestionsForKpAssembleSupplement()`.
|
|
|
+
|
|
|
+Look for the section where questions are merged/gathered and about to enter mastery-based filtering. Add the resolver call at that point:
|
|
|
+
|
|
|
+```php
|
|
|
+// Apply calibrated difficulty before mastery filtering and distribution
|
|
|
+$allQuestions = app(QuestionDifficultyResolver::class)->applyCalibratedDifficulty($allQuestions);
|
|
|
+```
|
|
|
+
|
|
|
+The exact variable name depends on the code structure -- it could be `$questions`, `$allQuestions`, `$candidateQuestions`, or similar. Read the code flow carefully:
|
|
|
+
|
|
|
+1. `getQuestionsFromBank()` returns raw questions from DB (line ~1734)
|
|
|
+2. These questions may be supplemented with `fetchQuestionsForKpAssembleSupplement()` (line ~1650)
|
|
|
+3. Then `selectQuestionsByMastery()` (line ~2097) filters/reorders them
|
|
|
+4. Then difficulty distribution is applied
|
|
|
+
|
|
|
+Insert the resolver call AFTER steps 1-2 (all questions gathered) and BEFORE step 3 (mastery filtering). This ensures calibrated difficulty is used for both mastery-based selection and difficulty distribution.
|
|
|
+
|
|
|
+Use `app(QuestionDifficultyResolver::class)` for container resolution -- same pattern as the job. Do NOT add constructor injection because the service already has an optional-parameter constructor pattern that would need careful handling.
|
|
|
+
|
|
|
+CRITICAL: Apply the resolver ONLY ONCE per question batch. Do not apply it multiple times -- the resolver queries the DB each time. Identify the single merge point where all candidate questions are collected and apply there.
|
|
|
+ </action>
|
|
|
+ <verify>
|
|
|
+ <automated>cd /Volumes/T9/code/math_cms && grep -n 'applyCalibratedDifficulty' app/Services/LearningAnalyticsService.php && grep -n "enable_difficulty_distribution.*\?\?" app/Services/LearningAnalyticsService.php</automated>
|
|
|
+ </verify>
|
|
|
+ <done>
|
|
|
+ - LearningAnalyticsService imports QuestionDifficultyResolver
|
|
|
+ - Calibrated difficulty is applied to the full question pool before mastery filtering and distribution
|
|
|
+ - Questions from getQuestionsFromBank() and fetchQuestionsForKpAssembleSupplement() both get calibrated difficulty
|
|
|
+ - enable_difficulty_distribution defaults to true (not false)
|
|
|
+ - Distribution is active even when ExamTypeStrategy::buildParams() does not run or fails
|
|
|
+ - Resolver is called exactly once per assembly, not multiple times
|
|
|
+ </done>
|
|
|
+</task>
|
|
|
+
|
|
|
+</tasks>
|
|
|
+
|
|
|
+<threat_model>
|
|
|
+## Trust Boundaries
|
|
|
+
|
|
|
+| Boundary | Description |
|
|
|
+|----------|-------------|
|
|
|
+| Assembly pipeline -> question_difficulty_calibrations | Read-only lookup of calibrated difficulty values |
|
|
|
+| Assembly pipeline -> paper_questions | Write with new difficulty_source field |
|
|
|
+
|
|
|
+## STRIDE Threat Register
|
|
|
+
|
|
|
+| Threat ID | Category | Component | Disposition | Mitigation Plan |
|
|
|
+|-----------|----------|-----------|-------------|-----------------|
|
|
|
+| T-02-01 | T | AssembleExamTaskJob | accept | Resolver reads from calibration table; no user input reaches the DB query. Question IDs come from prior validated queries |
|
|
|
+| T-02-02 | I | difficulty_source field | accept | Field is set server-side only, never from user input. Values are hardcoded 'calibrated' or 'original' |
|
|
|
+| T-02-03 | D | Migration | mitigate | Migration uses hasTable() guard to prevent failure on missing table. Column is nullable with default to avoid breaking existing rows |
|
|
|
+| T-02-04 | E | app() resolution | accept | QuestionDifficultyResolver is stateless with no external dependencies. Container resolution is safe |
|
|
|
+</threat_model>
|
|
|
+
|
|
|
+<verification>
|
|
|
+1. `php artisan migrate` runs without errors, adding difficulty_source column
|
|
|
+2. `grep -c 'applyCalibratedDifficulty' app/Jobs/AssembleExamTaskJob.php` returns >= 1
|
|
|
+3. `grep -c 'applyCalibratedDifficulty' app/Services/LearningAnalyticsService.php` returns >= 1
|
|
|
+4. `grep "enable_difficulty_distribution.*\?\?" app/Services/LearningAnalyticsService.php` shows `?? true`
|
|
|
+5. No double-application: count of applyCalibratedDifficulty calls in each file matches expectation (1 per file)
|
|
|
+6. `grep -r "difficulty_source" database/migrations/2026_04_17_100000_add_difficulty_source_to_paper_questions_table.php` confirms column exists
|
|
|
+</verification>
|
|
|
+
|
|
|
+<success_criteria>
|
|
|
+1. AssembleExamTaskJob::hydrateQuestions() applies calibrated difficulty before returning questions (ASM-01, ASM-02)
|
|
|
+2. LearningAnalyticsService::generateIntelligentExam() applies calibrated difficulty to the full candidate pool before mastery filtering (ASM-01)
|
|
|
+3. Questions without calibration data retain original difficulty unchanged (ASM-02)
|
|
|
+4. Difficulty distribution defaults to enabled (ASM-03)
|
|
|
+5. difficulty_source column exists in paper_questions table (partial ASM-04 -- column ready for persistence in Plan 02)
|
|
|
+</success_criteria>
|
|
|
+
|
|
|
+<output>
|
|
|
+After completion, create `.planning/phases/02-assembly-integration/02-01-SUMMARY.md`
|
|
|
+</output>
|