feat(effort): VPR-42 - Harvest Import process #96
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Contributor
rlorenzo
commented
Jan 17, 2026
rlorenzo
commented
- Add multi-phase harvest importing instructors, courses, and effort records from CREST, Banner, and Clinical Scheduler sources
- Implement SSE progress streaming and preview/diff UI before commit
- Add transactional harvest with rollback on failure
- Fix time parser for military time formats (e.g., "1430")
- Skip instructors without valid VIPER person records
- Use generic error messages to prevent info disclosure
- Add CrestContext/DictionaryContext for cross-database queries
- Split types/index.ts into domain-specific modules
- Add multi-phase harvest importing instructors, courses, and effort records from CREST, Banner, and Clinical Scheduler sources - Implement SSE progress streaming and preview/diff UI before commit - Add transactional harvest with rollback on failure - Fix time parser for military time formats (e.g., "1430") - Skip instructors without valid VIPER person records - Use generic error messages to prevent info disclosure - Add CrestContext/DictionaryContext for cross-database queries - Split types/index.ts into domain-specific modules
![github-code-quality[bot]](https://avatars.githubusercontent.com/u/9919?s=60&v=4){kind=small}
| var result = HarvestTimeParser.ParseTimeString(input); | ||
|
|
||
| Assert.NotNull(result); | ||
| Assert.Equal(expectedHour, result.Value.Hours); |
| var result = HarvestTimeParser.ParseTimeString(input); | ||
|
|
||
| Assert.NotNull(result); | ||
| Assert.Equal(expectedHour, result.Value.Hours); |
| var result = HarvestTimeParser.ParseTimeString(input); | ||
|
|
||
| Assert.NotNull(result); | ||
| Assert.Equal(expectedHour, result.Value.Hours); |
|
|
||
| Assert.NotNull(result); | ||
| // TimeSpan(24, 0, 0) stores as 1 day, 0 hours - check TotalHours | ||
| Assert.Equal(24, result.Value.TotalHours); |
Comment on lines
+95
to
+114
| foreach (var schedule in clinicalData) | ||
| { | ||
| if (string.IsNullOrEmpty(schedule.MothraId)) continue; | ||
|
|
||
| if (!personWeekCourses.ContainsKey(schedule.MothraId)) | ||
| { | ||
| personWeekCourses[schedule.MothraId] = new Dictionary<int, List<string>>(); | ||
| } | ||
|
|
||
| if (!personWeekCourses[schedule.MothraId].ContainsKey(schedule.WeekId)) | ||
| { | ||
| personWeekCourses[schedule.MothraId][schedule.WeekId] = []; | ||
| } | ||
|
|
||
| var courseKey = $"{schedule.SubjCode} {schedule.CrseNumb}"; | ||
| if (!personWeekCourses[schedule.MothraId][schedule.WeekId].Contains(courseKey)) | ||
| { | ||
| personWeekCourses[schedule.MothraId][schedule.WeekId].Add(courseKey); | ||
| } | ||
| } |
Comment on lines
+119
to
+149
| foreach (var schedule in clinicalData) | ||
| { | ||
| if (string.IsNullOrEmpty(schedule.MothraId) || string.IsNullOrEmpty(schedule.SubjCode)) continue; | ||
|
|
||
| var courseKey = $"{schedule.SubjCode} {schedule.CrseNumb}"; | ||
| var effortKey = $"{schedule.MothraId}-{courseKey}"; | ||
|
|
||
| // Check if this course is the priority for this person/week | ||
| var weekCourses = personWeekCourses[schedule.MothraId][schedule.WeekId]; | ||
| var priorityCourse = GetPriorityCourse(weekCourses); | ||
|
|
||
| if (courseKey != priorityCourse) continue; | ||
|
|
||
| if (!effortByPersonCourse.ContainsKey(effortKey)) | ||
| { | ||
| effortByPersonCourse[effortKey] = new HarvestRecordPreview | ||
| { | ||
| MothraId = schedule.MothraId, | ||
| PersonName = schedule.PersonName ?? "", | ||
| Crn = "", | ||
| CourseCode = courseKey, | ||
| EffortType = EffortConstants.ClinicalEffortType, | ||
| Weeks = 0, | ||
| RoleId = EffortConstants.ClinicalInstructorRoleId, | ||
| RoleName = "Instructor", | ||
| Source = EffortConstants.SourceClinical | ||
| }; | ||
| } | ||
|
|
||
| effortByPersonCourse[effortKey].Weeks = (effortByPersonCourse[effortKey].Weeks ?? 0) + 1; | ||
| } |
Comment on lines
+95
to
+102
| foreach (var course in context.Preview.CrestCourses) | ||
| { | ||
| if (rosterSummaries.TryGetValue(course.Crn, out var summary)) | ||
| { | ||
| course.Units = (decimal)summary.Units; | ||
| course.Enrollment = summary.Enrollment; | ||
| } | ||
| } |
Comment on lines
+193
to
+200
| foreach (var effort in context.Preview.CrestEffort) | ||
| { | ||
| var result = await ImportEffortRecordAsync(effort, context, ct); | ||
| if (result.Record != null && result.Preview != null) | ||
| { | ||
| context.CreatedRecords.Add((result.Record, result.Preview)); | ||
| } | ||
| } |
Comment on lines
+171
to
+178
| foreach (var effort in context.Preview.NonCrestEffort) | ||
| { | ||
| var result = await ImportEffortRecordAsync(effort, context, ct); | ||
| if (result.Record != null && result.Preview != null) | ||
| { | ||
| context.CreatedRecords.Add((result.Record, result.Preview)); | ||
| } | ||
| } |
Comment on lines
+152
to
+157
| catch (Exception ex) | ||
| { | ||
| _logger.LogError(ex, "Error during harvest for term {TermCode}", termCode); | ||
| await channel.Writer.WriteAsync(HarvestProgressEvent.Failed("An unexpected error occurred during harvest."), ct); | ||
| channel.Writer.Complete(); | ||
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.