Revamped final grade export

server/controllers/admin.py

@@ -414,13 +414,15 @@ def export_grades_job(cid):
     courses, current_course = get_courses(cid)
 
     form = forms.ExportGradesForm(current_course.assignments)
+
     if form.validate_on_submit():
         job = jobs.enqueue_job(
             export_grades.export_grades,
             description="Export Grades for {}".format(current_course.offering),
             timeout=2 * 60 * 60, # 1 hour
             course_id=cid,
             result_kind='link',
+            selected_assignments=form.included.data,
             # no arguments
         )
         return redirect(url_for('.course_job', cid=cid, job_id=job.id))
@@ -907,7 +909,7 @@ def assign_grading(cid, aid):
         data = assign.course_submissions()
         backups = set(b['backup']['id'] for b in data if b['backup'])
         students = set(b['user']['id'] for b in data if b['backup'])
-        
+
         tasks = GradingTask.create_staff_tasks(backups, selected_users, aid, cid,
                                                form.kind.data)
 
@@ -1444,7 +1446,7 @@ def client(client_id):
 def student_view(cid, email):
     form = forms.EnrollmentForm()
     if form.validate_on_submit():
-        if form.email.data != email:         
+        if form.email.data != email:
             user = User.lookup(email)
             new_email = form.email.data
 
@@ -1459,7 +1461,7 @@ def student_view(cid, email):
             except Forbidden as e:
                 flash(e.description, 'error')
                 return redirect(request.url)
-                
+
             Enrollment.enroll_from_form(cid, form)
             return redirect(url_for("admin.student_view", cid = cid, email = new_email), code=301)
         else:

server/forms.py

@@ -702,13 +702,16 @@ class EffortGradingForm(BaseForm):
                     description="Decimal ratio that is multiplied to the final score of a late submission.")
 
 class ExportGradesForm(BaseForm):
-    included = MultiCheckboxField('Included Assignments', description='Assignments with any published scores are checked by default')
+    included = MultiCheckboxField('Included Assignments', description='You can only export assignments with published scores')
+    export_submit_times = BooleanField('Export submission times', default=False)
 
     def __init__(self, assignments):
         super().__init__()
 
-        self.included.choices = [(str(a.id), a.display_name) for a in assignments]
-        self.included.data = [str(a.id) for a in assignments if a.published_scores]
+        self.included.choices = [(str(a.id), a.display_name) for a in assignments if a.published_scores]
+
+        if self.included.data is None:
+            self.included.data = [str(a.id) for a in assignments if a.published_scores]
 
     def validate(self):
         return super().validate() and len(self.included.data) > 0

server/jobs/export_grades.py

@@ -1,107 +1,196 @@
 import io
 import csv
 import datetime as dt
+from collections import defaultdict
 
 from server import jobs
-from server.models import Course, Enrollment, ExternalFile, db
+from server.models import (
+    Course,
+    Enrollment,
+    ExternalFile,
+    db,
+    GroupMember,
+    Score,
+    Assignment,
+    Backup,
+)
 from server.utils import encode_id, local_time
 from server.constants import STUDENT_ROLE
 
-TOTAL_KINDS = 'effort total regrade'.split()
-COMP_KINDS = 'composition revision'.split()
+TOTAL_KINDS = "effort total regrade".split()
+COMP_KINDS = "composition revision".split()
+
 
 def score_grabber(scores, kinds):
     return [scores.pop(kind.lower(), 0) for kind in kinds]
 
+
 def scores_checker(scores, kinds):
     return any(kind.lower() in scores for kind in kinds)
 
+
 def score_policy(scores):
     if scores_checker(scores, TOTAL_KINDS):
         total_score = max(score_grabber(scores, TOTAL_KINDS))
-        scores['total'] = total_score
+        scores["total"] = total_score
     if scores_checker(scores, COMP_KINDS):
         composition_score = max(score_grabber(scores, COMP_KINDS))
-        scores['composition'] = composition_score
+        scores["composition"] = composition_score
     return scores
 
 
 def get_score_types(assignment):
     types = []
     scores = [s.lower() for s in assignment.published_scores]
     if scores_checker(scores, TOTAL_KINDS):
-        types.append('total')
+        types.append("total")
     if scores_checker(scores, COMP_KINDS):
-        types.append('composition')
-    if scores_checker(scores, ['checkpoint 1']):
-        types.append('checkpoint 1')
-    if scores_checker(scores, ['checkpoint 2']):
-        types.append('checkpoint 2')
+        types.append("composition")
+    if scores_checker(scores, ["checkpoint 1"]):
+        types.append("checkpoint 1")
+    if scores_checker(scores, ["checkpoint 2"]):
+        types.append("checkpoint 2")
     return types
 
+
 def get_headers(assignments):
-    headers = ['Email', 'SID']
+    headers = ["Email", "SID"]
     new_assignments = []
     for assignment in assignments:
-        new_headers = ['{} ({})'.format(assignment.display_name, score_type.title()) for
-                score_type in get_score_types(assignment)]
+        new_headers = [
+            "{} ({})".format(assignment.display_name, score_type.title())
+            for score_type in get_score_types(assignment)
+        ]
         if new_headers:
             new_assignments.append(assignment)
             headers.extend(new_headers)
     return headers, new_assignments
 
-def export_student_grades(student, assignments):
+
+def collect_records(user_ids, assignments):
+    all_records = {}
+
+    for assign in assignments:
+        raw_assign_records = (
+            db.session.query(Score, Backup)
+            .join(Backup, Backup.id == Score.backup_id)
+            .filter(
+                Score.user_id.in_(user_ids),
+                Score.assignment_id == assign.id,
+                Score.archived == False,
+            )
+            .all()
+        )
+
+        members = GroupMember.query.filter(
+            GroupMember.assignment_id == assign.id, GroupMember.status == "active"
+        ).all()
+
+        group_lookup = {}
+        for member in members:
+            if member.group_id not in group_lookup:
+                group_lookup[member.group_id] = []
+            group_lookup[member.group_id].append(member.user_id)
+
+        gen = lambda: [None, None]
+        key = lambda a: float("-inf") if a[0] is None else a[0].score
+
+        assign_records = defaultdict(lambda: defaultdict(gen))
+
+        for record in raw_assign_records:
+            score = record[0]
+            assign_records[score.user_id][score.kind] = max(
+                record, assign_records[score.user_id][score.kind], key=key
+            )
+
+        for group in group_lookup.values():
+            best_scores = defaultdict(gen)
+            for user_id in group:
+                for kind, score in assign_records[user_id].items():
+                    best_scores[kind] = max(best_scores[kind], score, key=key)
+            for user_id in group:
+                assign_records[user_id] = best_scores
+
+        all_records[assign.id] = assign_records
+
+    return all_records
+
+
+def export_student_grades(student, assignments, all_records, *, export_submit_time):
     student_row = [student.user.email, student.sid]
     for assign in assignments:
-        status = assign.user_status(student.user)
-        scores = {s.kind.lower(): s.score for s in status.scores}
-        scores = score_policy(scores)
+        scores_for_each_kind = all_records[assign.id][student.user.id]
+        scores = score_policy(
+            {
+                kind: score.score
+                for kind, (score, backup) in scores_for_each_kind.items()
+            }
+        )
         score_types = get_score_types(assign)
         for score_type in score_types:
             if score_type in scores:
                 student_row.append(scores[score_type])
             else:
                 student_row.append(0)
+
     return student_row
 
 
 @jobs.background_job
-def export_grades():
+def export_grades(selected_assignments):
     logger = jobs.get_job_logger()
     current_user = jobs.get_current_job().user
     course = Course.query.get(jobs.get_current_job().course_id)
-    assignments = course.assignments
-    students = (Enrollment.query
-      .options(db.joinedload('user'))
-      .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course)
-      .all())
+    assignments = [
+        Assignment.query.get(int(assign_id)) for assign_id in selected_assignments
+    ]
+    students = (
+        Enrollment.query.options(db.joinedload("user"))
+        .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course)
+        .all()
+    )
 
-    headers, assignments = get_headers(assignments)
+    headers, assignments = get_headers(
+        assignments
+    )
     logger.info("Using these headers:")
     for header in headers:
-        logger.info('\t' + header)
-    logger.info('')
+        logger.info("\t" + header)
+    logger.info("")
 
     total_students = len(students)
+
+    users = [student.user for student in students]
+    user_ids = [user.id for user in users]
+
+    all_records = collect_records(user_ids, assignments)
+
     with io.StringIO() as f:
         writer = csv.writer(f)
-        writer.writerow(headers) # write headers
+        writer.writerow(headers)  # write headers
 
         for i, student in enumerate(students, start=1):
-            row = export_student_grades(student, assignments)
+            row = export_student_grades(
+                student, assignments, all_records
+            )
             writer.writerow(row)
             if i % 50 == 0:
-                logger.info('Exported {}/{}'.format(i, total_students))
+                logger.info("Exported {}/{}".format(i, total_students))
         f.seek(0)
-        created_time = local_time(dt.datetime.now(), course, fmt='%b-%-d %Y at %I-%M%p')
-        csv_filename = '{course_name} Grades ({date}).csv'.format(
-                course_name=course.display_name, date=created_time)
+        created_time = local_time(dt.datetime.now(), course, fmt="%b-%-d %Y at %I-%M%p")
+        csv_filename = "{course_name} Grades ({date}).csv".format(
+            course_name=course.display_name, date=created_time
+        )
         # convert to bytes for csv upload
-        csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8'))
-        upload = ExternalFile.upload(csv_bytes, user_id=current_user.id, name=csv_filename,
-                         course_id=course.id,
-                         prefix='jobs/exports/{}/'.format(course.offering))
+        csv_bytes = io.BytesIO(bytearray(f.read(), "utf-8"))
+        upload = ExternalFile.upload(
+            csv_bytes,
+            user_id=current_user.id,
+            name=csv_filename,
+            course_id=course.id,
+            prefix="jobs/exports/{}/".format(course.offering),
+        )
 
-    logger.info('\nDone!\n')
+    logger.info("\nDone!\n")
     logger.info("Saved as: {0}".format(upload.object_name))
     return "/files/{0}".format(encode_id(upload.id))

server/templates/staff/jobs/export_grades.html

@@ -30,6 +30,7 @@ <h3 class="box-title">Export Grades for {{ course.display_name }} </h3>
         <div class="box-body">
           {% call forms.render_form(form, action_text='Export Grades', class_='form') %}
             {{ forms.render_field(form.included, required='true', class_="checkbox-list") }}
+            {{ forms.render_checkbox_field(form.export_submit_times) }}
           {% endcall %}
         </div>
       </div>