Merge pull request #36 from dotkom/feat/improve-sync

Improve sync and preserve karstat data

clients/course_pages.py

@@ -14,6 +14,7 @@ class CoursePagesClient(Client):
         "se engelsk utgave",
         "see engelsk version",
         "see english text",
+        "se emnets engelske nettside",
     ]
 
     def extract_div_content(self, soup, div_id):
@@ -29,10 +30,12 @@ def extract_div_content(self, soup, div_id):
         for element in div.children:
             if element.name:
                 if element.name == "p":
-                    result.append(pDelimiter + element.get_text(strip=True))
+                    result.append(
+                        pDelimiter + element.get_text(strip=True, separator="\n")
+                    )
                 elif element.name == "ul" or element.name == "ol":
                     result.extend(
-                        liDelimiter + li.get_text(strip=True)
+                        liDelimiter + li.get_text(strip=True, separator="\n")
                         for li in element.find_all("li")
                     )
             elif element.string:
@@ -87,6 +90,8 @@ def extract_has_digital_exam(self, soup):
     def get_study_level_from_description(description: str):
         if description == "Doktorgrads nivå":
             return 900
+        elif description == "Videreutdanning høyere grad":
+            return 850
         elif description == "Videreutdanning lavere grad":
             return 800
         elif description == "Høyere grads nivå":
@@ -203,6 +208,8 @@ def get_course_data(self, code, year: int = None):
             classes = undervisning.get_text().split("Undervises")
             try:
                 place = undervisning.get_text().split("Sted:")[1].strip()
+                # Remove whitespace
+                place = " ".join(place.split()).replace(" ,", ",")
             except IndexError:
                 print("Cannot get place")
             for elements in classes:

clients/nsd.py

@@ -5,6 +5,7 @@
 from .client import Client
 
 from grades.models import Semester, Course, Grade
+from grades.utils import update_course_stats
 
 """
 API documentation can be found at:
@@ -128,14 +129,16 @@ def resolve_result_for_grade(self, results, letter: str):
         if len(grade_results) == 0:
             return 0
         elif len(grade_results) > 1:
-            raise Exception("Found more than a single grade entry for a course")
+            # Some codes have different course versions like -2. Use the highest version
+            grade_results.sort(key=lambda r: r["Emnekode"], reverse=True)
 
         grade_result = grade_results[0]
         return int(grade_result.get("Antall kandidater totalt"))
 
     def build_grade_data_from_results(
         self, results, course_code: str, year: int, semester: Semester
     ):
+        average_grade = 0
         passed = self.resolve_result_for_grade(results, "G")
         failed = self.resolve_result_for_grade(results, "H")
         a = self.resolve_result_for_grade(results, "A")
@@ -147,10 +150,27 @@ def build_grade_data_from_results(
 
         student_count = a + b + c + d + e + f
 
-        if student_count > 0:
-            average_grade = (a * 5.0 + b * 4 + c * 3 + d * 2 + e) / student_count
-        else:
+        is_pass_fail = any(map(lambda number: number != 0, [passed, failed]))
+        is_graded = any(map(lambda number: number != 0, [a, b, c, d, e, f]))
+
+        if is_pass_fail and is_graded:
+            print(
+                "Course is both pass/failed and graded by letters. Ignoring pass/fail"
+            )
+            is_pass_fail = False
+
+        if is_pass_fail:
             average_grade = 0
+            a = 0
+            b = 0
+            c = 0
+            d = 0
+            e = 0
+            f = failed
+        else:
+            if student_count > 0:
+                average_grade = (a * 5.0 + b * 4 + c * 3 + d * 2 + e) / student_count
+            passed = 0
 
         course_id = Course.all_objects.filter(code=course_code).values("id").first()
         if course_id:
@@ -164,7 +184,7 @@ def build_grade_data_from_results(
             "c": c,
             "d": d,
             "e": e,
-            "f": f + failed,  # Combine data for semesters with both pass/fail and graded exams
+            "f": f,
             "average_grade": average_grade,
             "passed": passed,
             "course_id": course_id,
@@ -199,6 +219,8 @@ def build_grade_from_data(self, grade_data):
                 year=year,
             ).update(**grade_data)
             grade.refresh_from_db()
+
+            update_course_stats(grade.course)
         except Grade.DoesNotExist:
             grade = Grade.objects.create(**grade_data)
 

grades/static/js/gradestats.js

@@ -4,14 +4,7 @@ $(function() {
 
     function createGraph(data){
         var s1, ticks, colors;
-        if (data.passed !== 0 && data.average_grade !== 0) {
-            s1 = [data.a, data.b, data.c, data.d, data.e, data.f, data.passed]
-            ticks = ['A', 'B', 'C', 'D', 'E', 'F', 'Bestått'];
-            colors = [ "#00CC00", "#00CC33", "#CCFF33", "#FFFF00", "#FF6600", "#CC0000", "#00CC00"];
-            barMargin = 2;
-            max = null;
-        }
-        else if(data.passed === 0){
+        if(data.passed === 0){
             s1 = [data.a, data.b, data.c, data.d, data.e, data.f]
             ticks = ['A', 'B', 'C', 'D', 'E', 'F'];
             colors = [ "#00CC00", "#00CC33", "#CCFF33", "#FFFF00", "#FF6600", "#CC0000"];

grades/utils.py

@@ -1,11 +1,11 @@
+from typing import List
 from django.core.mail import send_mail
 from django.conf import settings
 
-from .models import Course, Report
+from .models import Course, Report, Grade
 
 
-def calculate_average_grade(course: Course):
-    grades = course.grades.all()
+def calculate_average_grade(grades: List[Grade]):
     average = 0
     attendees = 0
     for grade in grades:
@@ -18,8 +18,7 @@ def calculate_average_grade(course: Course):
         return average
 
 
-def calculate_total_attendees(course: Course):
-    grades = course.grades.all()
+def calculate_total_attendees(grades: List[Grade]):
     attendees = 0
     for grade in grades:
         attendees += grade.attendee_count
@@ -28,10 +27,11 @@ def calculate_total_attendees(course: Course):
 
 
 def update_course_stats(course: Course):
-    average = calculate_average_grade(course)
+    grades = Grade.all_objects.filter(course_id=course.id)
+    average = calculate_average_grade(grades)
     course.average = average
 
-    attendee_count = calculate_total_attendees(course)
+    attendee_count = calculate_total_attendees(grades)
     course.attendee_count = attendee_count
 
     course.save()

requirements.txt

@@ -14,4 +14,4 @@ black==19.10b0
 sentry-sdk==0.14.3
 django-cors-headers==3.2.1
 mozilla-django-oidc==1.2.3
-uWSGI==2.0.18
+uWSGI==2.0.28

scripts/grades_sync.py

@@ -21,7 +21,7 @@
 
 
 def validate_course_code(code):
-    return re.match(r"^[a-zA-Z0-9-_\sæøå]+$", code)
+    return re.match(r"^[a-zA-Z0-9-_\sæøåÆØÅ]+$", code)
 
 
 def main():

services/grade_service.py

@@ -1,7 +1,7 @@
 from itertools import groupby
 from operator import itemgetter
 
-from grades.models import Semester, Course
+from grades.models import Semester, Course, Grade
 
 from clients.nsd import NSDGradeClient
 
@@ -31,21 +31,21 @@ def get_grade_data_for_semester(
 
         return grade_data
 
-    def get_grades_data_for_course(self, course_code, nhd_grades: List[dict] = None):
-        if nhd_grades is None:
-            nhd_grades = self.grade_client.get_grades_for_course(course_code)
+    def get_grades_data_for_course(self, course_code, nsd_grades: List[dict] = None):
+        if nsd_grades is None:
+            nsd_grades = self.grade_client.get_grades_for_course(course_code)
 
-        if not nhd_grades:
+        if not nsd_grades:
             return []
 
-        nhd_grades = sorted(nhd_grades, key=itemgetter("Årstall", "Semester"))
+        nsd_grades = sorted(nsd_grades, key=itemgetter("Årstall", "Semester"))
 
         grades_data = [
             self.get_grade_data_for_semester(
                 course_code, year, self.semester_map[semester_key], list(grade)
             )
             for (year, semester_key), grade in groupby(
-                nhd_grades, key=itemgetter("Årstall", "Semester")
+                nsd_grades, key=itemgetter("Årstall", "Semester")
             )
         ]
 
@@ -58,6 +58,9 @@ def create_or_update_grades_for_semester(
             raise ValueError(f"Course {course_code} does not exist")
 
         grade_data = self.get_grade_data_for_semester(course_code, year, semester)
+        grade_data = self.filter_out_conflicting_grades(
+            [grade_data], Grade.all_objects.filter(course__code=course_code)
+        )
 
         if not grade_data:
             print(
@@ -68,12 +71,15 @@ def create_or_update_grades_for_semester(
         return self.grade_client.build_grade_from_data(grade_data)
 
     def create_or_update_grades_for_course(
-        self, course_code, nhd_grades: List[dict] = None
+        self, course_code, nsd_grades: List[dict] = None
     ):
         if not Course.all_objects.filter(code=course_code).exists():
             raise ValueError(f"Course {course_code} does not exist")
 
-        grades_data = self.get_grades_data_for_course(course_code, nhd_grades)
+        grades_data = self.get_grades_data_for_course(course_code, nsd_grades)
+        grades_data = self.filter_out_conflicting_grades(
+            grades_data, Grade.all_objects.filter(course__code=course_code)
+        )
 
         if not grades_data:
             print(f"Course {course_code} has no grades")
@@ -83,3 +89,92 @@ def create_or_update_grades_for_course(
             self.grade_client.build_grade_from_data(grade_data)
             for grade_data in grades_data
         ]
+
+    def filter_out_conflicting_grades(
+        self, grades_data: List[dict], existing_grades_data: List[Grade]
+    ):
+        filtered_grades = []
+
+        grades_data.sort(key=lambda grade: int(grade["year"]))
+
+        summer_represents_spring, summer_represents_autumn = (
+            self.get_summer_semester_mapping(existing_grades_data)
+        )
+
+        # Allow overriding nsd data
+        last_year_with_karstat_data = 2021
+
+        for year, grades_by_year in groupby(grades_data, key=itemgetter("year")):
+            grades_by_year = list(grades_by_year)
+            existing_grades_by_year = [
+                grade for grade in existing_grades_data if grade.year == int(year)
+            ]
+
+            if int(year) > last_year_with_karstat_data or not existing_grades_by_year:
+                filtered_grades.extend(grades_by_year)
+                continue
+
+            existing_grades_semesters = {
+                grade.semester for grade in existing_grades_by_year
+            }
+            has_existing_summer_grades = "SUMMER" in existing_grades_semesters
+            has_existing_spring_grades = "SPRING" in existing_grades_semesters
+            has_existing_autumn_grades = "AUTUMN" in existing_grades_semesters
+
+            grades_by_semester = {
+                semester: [
+                    grade for grade in grades_by_year if grade["semester"] == semester
+                ]
+                for semester in ["SPRING", "AUTUMN", "SUMMER"]
+            }
+
+            if (
+                grades_by_semester["SPRING"]
+                and not has_existing_spring_grades
+                and (not has_existing_summer_grades or summer_represents_autumn)
+            ):
+                filtered_grades.extend(grades_by_semester["SPRING"])
+
+            if (
+                grades_by_semester["AUTUMN"]
+                and not has_existing_autumn_grades
+                and (not has_existing_summer_grades or summer_represents_spring)
+            ):
+                filtered_grades.extend(grades_by_semester["AUTUMN"])
+
+            if grades_by_semester["SUMMER"] and not has_existing_summer_grades:
+                filtered_grades.extend(grades_by_semester["SUMMER"])
+
+        return filtered_grades
+
+    def get_summer_semester_mapping(self, course_grades: List[Grade]):
+        distinct_years = set(grade.year for grade in course_grades)
+
+        UNKNOWN = None
+
+        summer_represents_spring = UNKNOWN
+        summer_represents_autumn = UNKNOWN
+
+        for year in distinct_years:
+            semesters = {
+                grade.semester for grade in course_grades if grade.year == year
+            }
+
+            has_spring = "SPRING" in semesters
+            has_summer = "SUMMER" in semesters
+            has_autumn = "AUTUMN" in semesters
+
+            if not has_summer:
+                continue
+
+            if has_autumn:
+                summer_represents_autumn = False
+            elif has_spring and summer_represents_autumn is UNKNOWN:
+                summer_represents_autumn = True
+
+            if has_spring:
+                summer_represents_spring = False
+            elif has_autumn and summer_represents_spring is UNKNOWN:
+                summer_represents_spring = True
+
+        return summer_represents_spring, summer_represents_autumn