From 23cde96ab5addb268ea5eda3df2d85ab91b159d1 Mon Sep 17 00:00:00 2001 From: rahularya Date: Tue, 31 Dec 2019 22:07:42 +0800 Subject: [PATCH 1/8] Experimental faster grade export. --- server/jobs/export_grades.py | 57 ++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index 94222240e..e32b0ba76 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -1,9 +1,12 @@ import io import csv import datetime as dt +from collections import defaultdict + +from sqlalchemy import func from server import jobs -from server.models import Course, Enrollment, ExternalFile, db +from server.models import Course, Enrollment, ExternalFile, db, GroupMember, Score from server.utils import encode_id, local_time from server.constants import STUDENT_ROLE @@ -50,11 +53,11 @@ def get_headers(assignments): headers.extend(new_headers) return headers, new_assignments -def export_student_grades(student, assignments): + +def export_student_grades(student, assignments, all_scores): student_row = [student.user.email, student.sid] for assign in assignments: - status = assign.user_status(student.user) - scores = {s.kind.lower(): s.score for s in status.scores} + scores = all_scores[assign.id][student.user.id] scores = score_policy(scores) score_types = get_score_types(assign) for score_type in score_types: @@ -83,12 +86,56 @@ def export_grades(): logger.info('') total_students = len(students) + + users = [student.user for student in students] + user_ids = [user.id for user in users] + + all_scores = {} + + for assign in assignments: + scores = ( + db.session.query(Score.user_id, Score.kind, func.max(Score.score)) + .filter( + Score.user_id.in_(user_ids), + Score.assignment_id == assign.id, + Score.archived == False, + ) + .group_by(Score.user_id, Score.kind) + .order_by(Score.score) + .all() + ) + + members = GroupMember.query.filter( + GroupMember.assignment_id == assign.id, + GroupMember.status == 'active' + ).all() + + group_lookup = {} + for member in members: + if member.group_id not in group_lookup: + group_lookup[member.group_id] = [] + group_lookup[member.group_id].append(member.user_id) + + user_scores = defaultdict(lambda: defaultdict(int)) + for user_id, kind, score in scores: + user_scores[user_id][kind] = score + + for group in group_lookup.values(): + best_scores = defaultdict(int) + for user_id in group: + for kind, score in user_scores[user_id].items(): + best_scores[kind] = max(best_scores[kind], score) + for user_id in group: + user_scores[user_id] = best_scores + + all_scores[assign.id] = user_scores + with io.StringIO() as f: writer = csv.writer(f) writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): - row = export_student_grades(student, assignments) + row = export_student_grades(student, assignments, all_scores) writer.writerow(row) if i % 50 == 0: logger.info('Exported {}/{}'.format(i, total_students)) From ed96913ad6d7270eed7f81b81d76175e3209d51a Mon Sep 17 00:00:00 2001 From: Kavi Gupta Date: Tue, 31 Dec 2019 17:30:45 -0800 Subject: [PATCH 2/8] factor out collection of all scores --- server/jobs/export_grades.py | 48 +++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index e32b0ba76..f022570e0 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -68,28 +68,7 @@ def export_student_grades(student, assignments, all_scores): return student_row -@jobs.background_job -def export_grades(): - logger = jobs.get_job_logger() - current_user = jobs.get_current_job().user - course = Course.query.get(jobs.get_current_job().course_id) - assignments = course.assignments - students = (Enrollment.query - .options(db.joinedload('user')) - .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) - .all()) - - headers, assignments = get_headers(assignments) - logger.info("Using these headers:") - for header in headers: - logger.info('\t' + header) - logger.info('') - - total_students = len(students) - - users = [student.user for student in students] - user_ids = [user.id for user in users] - +def collect_all_scores(assignments, user_ids): all_scores = {} for assign in assignments: @@ -129,6 +108,31 @@ def export_grades(): user_scores[user_id] = best_scores all_scores[assign.id] = user_scores + return all_scores + +@jobs.background_job +def export_grades(): + logger = jobs.get_job_logger() + current_user = jobs.get_current_job().user + course = Course.query.get(jobs.get_current_job().course_id) + assignments = course.assignments + students = (Enrollment.query + .options(db.joinedload('user')) + .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) + .all()) + + headers, assignments = get_headers(assignments) + logger.info("Using these headers:") + for header in headers: + logger.info('\t' + header) + logger.info('') + + total_students = len(students) + + users = [student.user for student in students] + user_ids = [user.id for user in users] + + all_scores = collect_all_scores(assignments, user_ids) with io.StringIO() as f: writer = csv.writer(f) From 047064f55aa5091919481c3c80695d7f7b28000b Mon Sep 17 00:00:00 2001 From: rahularya Date: Thu, 9 Jan 2020 05:08:13 +0800 Subject: [PATCH 3/8] experimental flag to show submission times --- server/controllers/admin.py | 9 ++-- server/forms.py | 5 +- server/jobs/export_grades.py | 54 ++++++++++++------- .../templates/staff/jobs/export_grades.html | 1 + 4 files changed, 46 insertions(+), 23 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index b615f4af3..ed5c9c46f 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -414,6 +414,7 @@ def export_grades_job(cid): courses, current_course = get_courses(cid) form = forms.ExportGradesForm(current_course.assignments) + if form.validate_on_submit(): job = jobs.enqueue_job( export_grades.export_grades, @@ -421,6 +422,8 @@ def export_grades_job(cid): timeout=2 * 60 * 60, # 1 hour course_id=cid, result_kind='link', + selected_assignments=form.included.data, + export_submit_time=form.export_submit_times.data, # no arguments ) return redirect(url_for('.course_job', cid=cid, job_id=job.id)) @@ -907,7 +910,7 @@ def assign_grading(cid, aid): data = assign.course_submissions() backups = set(b['backup']['id'] for b in data if b['backup']) students = set(b['user']['id'] for b in data if b['backup']) - + tasks = GradingTask.create_staff_tasks(backups, selected_users, aid, cid, form.kind.data) @@ -1444,7 +1447,7 @@ def client(client_id): def student_view(cid, email): form = forms.EnrollmentForm() if form.validate_on_submit(): - if form.email.data != email: + if form.email.data != email: user = User.lookup(email) new_email = form.email.data @@ -1459,7 +1462,7 @@ def student_view(cid, email): except Forbidden as e: flash(e.description, 'error') return redirect(request.url) - + Enrollment.enroll_from_form(cid, form) return redirect(url_for("admin.student_view", cid = cid, email = new_email), code=301) else: diff --git a/server/forms.py b/server/forms.py index ce8d4f7c7..fdfba802f 100644 --- a/server/forms.py +++ b/server/forms.py @@ -703,12 +703,15 @@ class EffortGradingForm(BaseForm): class ExportGradesForm(BaseForm): included = MultiCheckboxField('Included Assignments', description='Assignments with any published scores are checked by default') + export_submit_times = BooleanField('Export submission times', default=False) def __init__(self, assignments): super().__init__() self.included.choices = [(str(a.id), a.display_name) for a in assignments] - self.included.data = [str(a.id) for a in assignments if a.published_scores] + + if self.included.data is None: + self.included.data = [str(a.id) for a in assignments if a.published_scores] def validate(self): return super().validate() and len(self.included.data) > 0 diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index e32b0ba76..251e906b8 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -3,10 +3,8 @@ import datetime as dt from collections import defaultdict -from sqlalchemy import func - from server import jobs -from server.models import Course, Enrollment, ExternalFile, db, GroupMember, Score +from server.models import Course, Enrollment, ExternalFile, db, GroupMember, Score, Assignment, Backup from server.utils import encode_id, local_time from server.constants import STUDENT_ROLE @@ -42,7 +40,7 @@ def get_score_types(assignment): types.append('checkpoint 2') return types -def get_headers(assignments): +def get_headers(assignments, *, export_submit_time): headers = ['Email', 'SID'] new_assignments = [] for assignment in assignments: @@ -51,35 +49,47 @@ def get_headers(assignments): if new_headers: new_assignments.append(assignment) headers.extend(new_headers) + if export_submit_time: + headers.append('{} (Submitted At)'.format(assignment.display_name)) return headers, new_assignments -def export_student_grades(student, assignments, all_scores): +def export_student_grades(student, assignments, all_scores, *, export_submit_time): student_row = [student.user.email, student.sid] for assign in assignments: - scores = all_scores[assign.id][student.user.id] - scores = score_policy(scores) + scores_for_each_kind = all_scores[assign.id][student.user.id] + scores = score_policy({kind: score.score for kind, (score, backup) in scores_for_each_kind.items()}) score_types = get_score_types(assign) for score_type in score_types: if score_type in scores: student_row.append(scores[score_type]) else: student_row.append(0) + + if export_submit_time: + if scores_for_each_kind: + candidate_backup = next(iter(scores_for_each_kind.values()))[1] + if all(backup.submission_time == candidate_backup.submission_time for _, backup in scores_for_each_kind.values()): + student_row.append(candidate_backup.submission_time) + else: + student_row.append("Multiple Submissions Scored") + else: + student_row.append("No Submission") return student_row @jobs.background_job -def export_grades(): +def export_grades(selected_assignments, export_submit_time): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) - assignments = course.assignments + assignments = [Assignment.query.get(int(assign_id)) for assign_id in selected_assignments] students = (Enrollment.query .options(db.joinedload('user')) .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) .all()) - headers, assignments = get_headers(assignments) + headers, assignments = get_headers(assignments, export_submit_time=export_submit_time) logger.info("Using these headers:") for header in headers: logger.info('\t' + header) @@ -94,17 +104,18 @@ def export_grades(): for assign in assignments: scores = ( - db.session.query(Score.user_id, Score.kind, func.max(Score.score)) + db.session.query(Score, Backup) + .join(Backup, Backup.id == Score.backup_id) .filter( Score.user_id.in_(user_ids), Score.assignment_id == assign.id, Score.archived == False, ) - .group_by(Score.user_id, Score.kind) - .order_by(Score.score) .all() ) + logger.info(scores) + members = GroupMember.query.filter( GroupMember.assignment_id == assign.id, GroupMember.status == 'active' @@ -116,15 +127,20 @@ def export_grades(): group_lookup[member.group_id] = [] group_lookup[member.group_id].append(member.user_id) - user_scores = defaultdict(lambda: defaultdict(int)) - for user_id, kind, score in scores: - user_scores[user_id][kind] = score + gen = lambda: [None, None] + key = lambda a: float("-inf") if a[0] is None else a[0].score + + user_scores = defaultdict(lambda: defaultdict(gen)) + + for record in scores: + score = record[0] + user_scores[score.user_id][score.kind] = max(record, user_scores[score.user_id][score.kind], key=key) for group in group_lookup.values(): - best_scores = defaultdict(int) + best_scores = defaultdict(gen) for user_id in group: for kind, score in user_scores[user_id].items(): - best_scores[kind] = max(best_scores[kind], score) + best_scores[kind] = max(best_scores[kind], score, key=key) for user_id in group: user_scores[user_id] = best_scores @@ -135,7 +151,7 @@ def export_grades(): writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): - row = export_student_grades(student, assignments, all_scores) + row = export_student_grades(student, assignments, all_scores, export_submit_time=export_submit_time) writer.writerow(row) if i % 50 == 0: logger.info('Exported {}/{}'.format(i, total_students)) diff --git a/server/templates/staff/jobs/export_grades.html b/server/templates/staff/jobs/export_grades.html index 7f28e6780..ad9dabdf2 100644 --- a/server/templates/staff/jobs/export_grades.html +++ b/server/templates/staff/jobs/export_grades.html @@ -30,6 +30,7 @@

Export Grades for {{ course.display_name }}

{% call forms.render_form(form, action_text='Export Grades', class_='form') %} {{ forms.render_field(form.included, required='true', class_="checkbox-list") }} + {{ forms.render_checkbox_field(form.export_submit_times) }} {% endcall %}
From 166172cbf691a0182330133e6fbce6289ed07211 Mon Sep 17 00:00:00 2001 From: rahularya Date: Thu, 9 Jan 2020 05:13:39 +0800 Subject: [PATCH 4/8] fixed merge --- server/jobs/export_grades.py | 106 ++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 51 deletions(-) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index 0de8c7e51..a9a10a595 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -54,10 +54,58 @@ def get_headers(assignments, *, export_submit_time): return headers, new_assignments -def export_student_grades(student, assignments, all_scores, *, export_submit_time): +def collect_records(user_ids, assignments): + all_records = {} + + for assign in assignments: + raw_assign_records = ( + db.session.query(Score, Backup) + .join(Backup, Backup.id == Score.backup_id) + .filter( + Score.user_id.in_(user_ids), + Score.assignment_id == assign.id, + Score.archived == False, + ) + .all() + ) + + members = GroupMember.query.filter( + GroupMember.assignment_id == assign.id, + GroupMember.status == 'active' + ).all() + + group_lookup = {} + for member in members: + if member.group_id not in group_lookup: + group_lookup[member.group_id] = [] + group_lookup[member.group_id].append(member.user_id) + + gen = lambda: [None, None] + key = lambda a: float("-inf") if a[0] is None else a[0].score + + assign_records = defaultdict(lambda: defaultdict(gen)) + + for record in raw_assign_records: + score = record[0] + assign_records[score.user_id][score.kind] = max(record, assign_records[score.user_id][score.kind], key=key) + + for group in group_lookup.values(): + best_scores = defaultdict(gen) + for user_id in group: + for kind, score in assign_records[user_id].items(): + best_scores[kind] = max(best_scores[kind], score, key=key) + for user_id in group: + assign_records[user_id] = best_scores + + all_records[assign.id] = assign_records + + return all_records + + +def export_student_grades(student, assignments, all_records, *, export_submit_time): student_row = [student.user.email, student.sid] for assign in assignments: - scores_for_each_kind = all_scores[assign.id][student.user.id] + scores_for_each_kind = all_records[assign.id][student.user.id] scores = score_policy({kind: score.score for kind, (score, backup) in scores_for_each_kind.items()}) score_types = get_score_types(assign) for score_type in score_types: @@ -79,17 +127,17 @@ def export_student_grades(student, assignments, all_scores, *, export_submit_tim @jobs.background_job -def export_grades(): +def export_grades(selected_assignments, export_submit_time): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) - assignments = course.assignments + assignments = [Assignment.query.get(int(assign_id)) for assign_id in selected_assignments] students = (Enrollment.query .options(db.joinedload('user')) .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) .all()) - headers, assignments = get_headers(assignments) + headers, assignments = get_headers(assignments, export_submit_time=export_submit_time) logger.info("Using these headers:") for header in headers: logger.info('\t' + header) @@ -100,58 +148,14 @@ def export_grades(): users = [student.user for student in students] user_ids = [user.id for user in users] - all_scores = {} - - for assign in assignments: - scores = ( - db.session.query(Score, Backup) - .join(Backup, Backup.id == Score.backup_id) - .filter( - Score.user_id.in_(user_ids), - Score.assignment_id == assign.id, - Score.archived == False, - ) - .all() - ) - - logger.info(scores) - - members = GroupMember.query.filter( - GroupMember.assignment_id == assign.id, - GroupMember.status == 'active' - ).all() - - group_lookup = {} - for member in members: - if member.group_id not in group_lookup: - group_lookup[member.group_id] = [] - group_lookup[member.group_id].append(member.user_id) - - gen = lambda: [None, None] - key = lambda a: float("-inf") if a[0] is None else a[0].score - - user_scores = defaultdict(lambda: defaultdict(gen)) - - for record in scores: - score = record[0] - user_scores[score.user_id][score.kind] = max(record, user_scores[score.user_id][score.kind], key=key) - - for group in group_lookup.values(): - best_scores = defaultdict(gen) - for user_id in group: - for kind, score in user_scores[user_id].items(): - best_scores[kind] = max(best_scores[kind], score, key=key) - for user_id in group: - user_scores[user_id] = best_scores - - all_scores[assign.id] = user_scores + all_records = collect_records(user_ids, assignments) with io.StringIO() as f: writer = csv.writer(f) writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): - row = export_student_grades(student, assignments, all_scores, export_submit_time=export_submit_time) + row = export_student_grades(student, assignments, all_records, export_submit_time=export_submit_time) writer.writerow(row) if i % 50 == 0: logger.info('Exported {}/{}'.format(i, total_students)) From dd3f290242275931077ff85ae2630115d1706dc3 Mon Sep 17 00:00:00 2001 From: rahularya Date: Thu, 9 Jan 2020 05:15:31 +0800 Subject: [PATCH 5/8] style pass --- server/jobs/export_grades.py | 122 +++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 43 deletions(-) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index a9a10a595..0c91a2a08 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -4,26 +4,38 @@ from collections import defaultdict from server import jobs -from server.models import Course, Enrollment, ExternalFile, db, GroupMember, Score, Assignment, Backup +from server.models import ( + Course, + Enrollment, + ExternalFile, + db, + GroupMember, + Score, + Assignment, + Backup, +) from server.utils import encode_id, local_time from server.constants import STUDENT_ROLE -TOTAL_KINDS = 'effort total regrade'.split() -COMP_KINDS = 'composition revision'.split() +TOTAL_KINDS = "effort total regrade".split() +COMP_KINDS = "composition revision".split() + def score_grabber(scores, kinds): return [scores.pop(kind.lower(), 0) for kind in kinds] + def scores_checker(scores, kinds): return any(kind.lower() in scores for kind in kinds) + def score_policy(scores): if scores_checker(scores, TOTAL_KINDS): total_score = max(score_grabber(scores, TOTAL_KINDS)) - scores['total'] = total_score + scores["total"] = total_score if scores_checker(scores, COMP_KINDS): composition_score = max(score_grabber(scores, COMP_KINDS)) - scores['composition'] = composition_score + scores["composition"] = composition_score return scores @@ -31,26 +43,29 @@ def get_score_types(assignment): types = [] scores = [s.lower() for s in assignment.published_scores] if scores_checker(scores, TOTAL_KINDS): - types.append('total') + types.append("total") if scores_checker(scores, COMP_KINDS): - types.append('composition') - if scores_checker(scores, ['checkpoint 1']): - types.append('checkpoint 1') - if scores_checker(scores, ['checkpoint 2']): - types.append('checkpoint 2') + types.append("composition") + if scores_checker(scores, ["checkpoint 1"]): + types.append("checkpoint 1") + if scores_checker(scores, ["checkpoint 2"]): + types.append("checkpoint 2") return types + def get_headers(assignments, *, export_submit_time): - headers = ['Email', 'SID'] + headers = ["Email", "SID"] new_assignments = [] for assignment in assignments: - new_headers = ['{} ({})'.format(assignment.display_name, score_type.title()) for - score_type in get_score_types(assignment)] + new_headers = [ + "{} ({})".format(assignment.display_name, score_type.title()) + for score_type in get_score_types(assignment) + ] if new_headers: new_assignments.append(assignment) headers.extend(new_headers) if export_submit_time: - headers.append('{} (Submitted At)'.format(assignment.display_name)) + headers.append("{} (Submitted At)".format(assignment.display_name)) return headers, new_assignments @@ -60,18 +75,17 @@ def collect_records(user_ids, assignments): for assign in assignments: raw_assign_records = ( db.session.query(Score, Backup) - .join(Backup, Backup.id == Score.backup_id) - .filter( + .join(Backup, Backup.id == Score.backup_id) + .filter( Score.user_id.in_(user_ids), Score.assignment_id == assign.id, Score.archived == False, ) - .all() + .all() ) members = GroupMember.query.filter( - GroupMember.assignment_id == assign.id, - GroupMember.status == 'active' + GroupMember.assignment_id == assign.id, GroupMember.status == "active" ).all() group_lookup = {} @@ -87,7 +101,9 @@ def collect_records(user_ids, assignments): for record in raw_assign_records: score = record[0] - assign_records[score.user_id][score.kind] = max(record, assign_records[score.user_id][score.kind], key=key) + assign_records[score.user_id][score.kind] = max( + record, assign_records[score.user_id][score.kind], key=key + ) for group in group_lookup.values(): best_scores = defaultdict(gen) @@ -106,7 +122,12 @@ def export_student_grades(student, assignments, all_records, *, export_submit_ti student_row = [student.user.email, student.sid] for assign in assignments: scores_for_each_kind = all_records[assign.id][student.user.id] - scores = score_policy({kind: score.score for kind, (score, backup) in scores_for_each_kind.items()}) + scores = score_policy( + { + kind: score.score + for kind, (score, backup) in scores_for_each_kind.items() + } + ) score_types = get_score_types(assign) for score_type in score_types: if score_type in scores: @@ -117,7 +138,10 @@ def export_student_grades(student, assignments, all_records, *, export_submit_ti if export_submit_time: if scores_for_each_kind: candidate_backup = next(iter(scores_for_each_kind.values()))[1] - if all(backup.submission_time == candidate_backup.submission_time for _, backup in scores_for_each_kind.values()): + if all( + backup.submission_time == candidate_backup.submission_time + for _, backup in scores_for_each_kind.values() + ): student_row.append(candidate_backup.submission_time) else: student_row.append("Multiple Submissions Scored") @@ -131,17 +155,22 @@ def export_grades(selected_assignments, export_submit_time): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) - assignments = [Assignment.query.get(int(assign_id)) for assign_id in selected_assignments] - students = (Enrollment.query - .options(db.joinedload('user')) - .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) - .all()) - - headers, assignments = get_headers(assignments, export_submit_time=export_submit_time) + assignments = [ + Assignment.query.get(int(assign_id)) for assign_id in selected_assignments + ] + students = ( + Enrollment.query.options(db.joinedload("user")) + .filter(Enrollment.role == STUDENT_ROLE, Enrollment.course == course) + .all() + ) + + headers, assignments = get_headers( + assignments, export_submit_time=export_submit_time + ) logger.info("Using these headers:") for header in headers: - logger.info('\t' + header) - logger.info('') + logger.info("\t" + header) + logger.info("") total_students = len(students) @@ -152,23 +181,30 @@ def export_grades(selected_assignments, export_submit_time): with io.StringIO() as f: writer = csv.writer(f) - writer.writerow(headers) # write headers + writer.writerow(headers) # write headers for i, student in enumerate(students, start=1): - row = export_student_grades(student, assignments, all_records, export_submit_time=export_submit_time) + row = export_student_grades( + student, assignments, all_records, export_submit_time=export_submit_time + ) writer.writerow(row) if i % 50 == 0: - logger.info('Exported {}/{}'.format(i, total_students)) + logger.info("Exported {}/{}".format(i, total_students)) f.seek(0) - created_time = local_time(dt.datetime.now(), course, fmt='%b-%-d %Y at %I-%M%p') - csv_filename = '{course_name} Grades ({date}).csv'.format( - course_name=course.display_name, date=created_time) + created_time = local_time(dt.datetime.now(), course, fmt="%b-%-d %Y at %I-%M%p") + csv_filename = "{course_name} Grades ({date}).csv".format( + course_name=course.display_name, date=created_time + ) # convert to bytes for csv upload - csv_bytes = io.BytesIO(bytearray(f.read(), 'utf-8')) - upload = ExternalFile.upload(csv_bytes, user_id=current_user.id, name=csv_filename, - course_id=course.id, - prefix='jobs/exports/{}/'.format(course.offering)) + csv_bytes = io.BytesIO(bytearray(f.read(), "utf-8")) + upload = ExternalFile.upload( + csv_bytes, + user_id=current_user.id, + name=csv_filename, + course_id=course.id, + prefix="jobs/exports/{}/".format(course.offering), + ) - logger.info('\nDone!\n') + logger.info("\nDone!\n") logger.info("Saved as: {0}".format(upload.object_name)) return "/files/{0}".format(encode_id(upload.id)) From 0d53c1155bbf1d879bc50f85d977f423c43a8092 Mon Sep 17 00:00:00 2001 From: rahularya Date: Thu, 9 Jan 2020 05:16:33 +0800 Subject: [PATCH 6/8] Tweaked form to match actual behavior. --- server/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/forms.py b/server/forms.py index fdfba802f..1b6d0fda5 100644 --- a/server/forms.py +++ b/server/forms.py @@ -702,13 +702,13 @@ class EffortGradingForm(BaseForm): description="Decimal ratio that is multiplied to the final score of a late submission.") class ExportGradesForm(BaseForm): - included = MultiCheckboxField('Included Assignments', description='Assignments with any published scores are checked by default') + included = MultiCheckboxField('Included Assignments', description='You can only export assignments with published scores') export_submit_times = BooleanField('Export submission times', default=False) def __init__(self, assignments): super().__init__() - self.included.choices = [(str(a.id), a.display_name) for a in assignments] + self.included.choices = [(str(a.id), a.display_name) for a in assignments if a.published_scores] if self.included.data is None: self.included.data = [str(a.id) for a in assignments if a.published_scores] From 5e1d94ed25800c6c4730ec39947faff06b042388 Mon Sep 17 00:00:00 2001 From: rahularya Date: Thu, 9 Jan 2020 16:36:13 +0800 Subject: [PATCH 7/8] return min submission times --- server/jobs/export_grades.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index 0c91a2a08..a7cb752ba 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -137,14 +137,12 @@ def export_student_grades(student, assignments, all_records, *, export_submit_ti if export_submit_time: if scores_for_each_kind: - candidate_backup = next(iter(scores_for_each_kind.values()))[1] - if all( - backup.submission_time == candidate_backup.submission_time - for _, backup in scores_for_each_kind.values() - ): - student_row.append(candidate_backup.submission_time) - else: - student_row.append("Multiple Submissions Scored") + student_row.append( + min( + backup.submission_time + for _, backup in scores_for_each_kind.values() + ) + ) else: student_row.append("No Submission") return student_row From 05a3a4ffce67297f7acd8bafb055d33ec109428b Mon Sep 17 00:00:00 2001 From: rahularya Date: Mon, 24 Feb 2020 17:10:21 -0800 Subject: [PATCH 8/8] removed submission times from csv --- server/controllers/admin.py | 1 - server/jobs/export_grades.py | 20 ++++---------------- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/server/controllers/admin.py b/server/controllers/admin.py index ed5c9c46f..c02d7ed68 100644 --- a/server/controllers/admin.py +++ b/server/controllers/admin.py @@ -423,7 +423,6 @@ def export_grades_job(cid): course_id=cid, result_kind='link', selected_assignments=form.included.data, - export_submit_time=form.export_submit_times.data, # no arguments ) return redirect(url_for('.course_job', cid=cid, job_id=job.id)) diff --git a/server/jobs/export_grades.py b/server/jobs/export_grades.py index a7cb752ba..c5ab85f13 100644 --- a/server/jobs/export_grades.py +++ b/server/jobs/export_grades.py @@ -53,7 +53,7 @@ def get_score_types(assignment): return types -def get_headers(assignments, *, export_submit_time): +def get_headers(assignments): headers = ["Email", "SID"] new_assignments = [] for assignment in assignments: @@ -64,8 +64,6 @@ def get_headers(assignments, *, export_submit_time): if new_headers: new_assignments.append(assignment) headers.extend(new_headers) - if export_submit_time: - headers.append("{} (Submitted At)".format(assignment.display_name)) return headers, new_assignments @@ -135,21 +133,11 @@ def export_student_grades(student, assignments, all_records, *, export_submit_ti else: student_row.append(0) - if export_submit_time: - if scores_for_each_kind: - student_row.append( - min( - backup.submission_time - for _, backup in scores_for_each_kind.values() - ) - ) - else: - student_row.append("No Submission") return student_row @jobs.background_job -def export_grades(selected_assignments, export_submit_time): +def export_grades(selected_assignments): logger = jobs.get_job_logger() current_user = jobs.get_current_job().user course = Course.query.get(jobs.get_current_job().course_id) @@ -163,7 +151,7 @@ def export_grades(selected_assignments, export_submit_time): ) headers, assignments = get_headers( - assignments, export_submit_time=export_submit_time + assignments ) logger.info("Using these headers:") for header in headers: @@ -183,7 +171,7 @@ def export_grades(selected_assignments, export_submit_time): for i, student in enumerate(students, start=1): row = export_student_grades( - student, assignments, all_records, export_submit_time=export_submit_time + student, assignments, all_records ) writer.writerow(row) if i % 50 == 0: