diff --git a/.github/workflows/test_and_release.yaml b/.github/workflows/test_and_release.yaml new file mode 100644 index 0000000..ee90d27 --- /dev/null +++ b/.github/workflows/test_and_release.yaml @@ -0,0 +1,58 @@ +name: test_and_release + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + create_tag: + permissions: + contents: write + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + python-version: [ 3.11 ] + + name: A job to test the functionalities of the code + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Installing Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Installing this repo as a Python package + run: | + python3 -m pip install ${{ github.workspace }} + + - name: Installing GitHubApiHelper + run: | + python3 -m pip install git+https://github.com/zhenghaven/GitHubApiHelper.git@v0.1.3 + + - name: Get latest version + id: latest_ver + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + python3 -m GitHubApiHelper --auth-token \ + api_tags_latest_ver \ + --repo ${{ github.repository }} \ + -l $(python3 -m zyAPI --version) \ + --github-out + + - name: Create tag + if: ${{ startsWith(github.ref, 'refs/heads/main') && steps.latest_ver.outputs.remote != steps.latest_ver.outputs.all }} + uses: actions/github-script@v6 + with: + script: | + github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: 'refs/tags/${{ steps.latest_ver.outputs.allV }}', + sha: context.sha + }) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9aa3634 --- /dev/null +++ b/.gitignore @@ -0,0 +1,163 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +build/ +.DS_Store/ diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c64ae40 --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + + +from setuptools import setup +from setuptools import find_packages + +import zyAPI._Meta + + +setup( + name = zyAPI._Meta.PKG_NAME, + version = zyAPI._Meta.__version__, + packages = find_packages(where='.', exclude=['setup.py']), + url = 'https://github.com/lsd-ucsc/zyAPI', + license = zyAPI._Meta.PKG_LICENSE, + author = zyAPI._Meta.PKG_AUTHOR, + description = zyAPI._Meta.PKG_DESCRIPTION, + entry_points= { + 'console_scripts': [ + 'zyAPI=zyAPI.__main__:main', + ] + }, + install_requires=[ + 'requests==2.31.0', + 'pandas==2.2.1', + 'numpy==1.26.4', + ], +) diff --git a/zyAPI/Auth/Auth.py b/zyAPI/Auth/Auth.py new file mode 100644 index 0000000..0fc5d13 --- /dev/null +++ b/zyAPI/Auth/Auth.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +class Auth(object): + + def __init__(self) -> None: + super(Auth, self).__init__() + + def AddAuth(self, headers: dict) -> None: + raise NotImplementedError('AddAuth not implemented') diff --git a/zyAPI/Auth/Token.py b/zyAPI/Auth/Token.py new file mode 100644 index 0000000..10b8dac --- /dev/null +++ b/zyAPI/Auth/Token.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +from . import Auth + + +class Token(Auth.Auth): + + @classmethod + def FromFile(cls, path: str) -> 'Token': + with open(path, 'r') as f: + token = f.read().strip() + return cls(token) + + def __init__(self, token: str) -> None: + super(Token, self).__init__() + + self.token = token + + def AddAuth(self, headers: dict) -> None: + headers['Authorization'] = f'Bearer {self.token}' + diff --git a/zyAPI/Auth/__init__.py b/zyAPI/Auth/__init__.py new file mode 100644 index 0000000..97aa456 --- /dev/null +++ b/zyAPI/Auth/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + diff --git a/zyAPI/Due/Datetime.py b/zyAPI/Due/Datetime.py new file mode 100644 index 0000000..830feb3 --- /dev/null +++ b/zyAPI/Due/Datetime.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import datetime + +try: + import zoneinfo +except ImportError: + from backports import zoneinfo + + +class Datetime(object): + + def __init__(self, datetime: datetime.datetime) -> None: + super(Datetime, self).__init__() + + self.datetime = datetime + + def GetTimestampStr(self) -> str: + # format: '2024-03-08T07:59:59.999Z' + return self.datetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + def GetUtcTimestampStr(self) -> str: + utcDatetime = self.datetime.astimezone(zoneinfo.ZoneInfo('UTC')) + return utcDatetime.strftime('%Y-%m-%dT%H:%M:%S.%fZ') + + def GetTimezoneAbbr(self) -> str: + return self.datetime.strftime('%Z') + + def GetTimezoneOffsetMin(self) -> int: + return int(self.datetime.utcoffset().total_seconds()) // 60 + + def GetZyTimezoneOffsetMin(self) -> int: + return -1 * self.GetTimezoneOffsetMin() + + def GetReportNameSuffix(self) -> str: + # report_2024-01-20_0759_PST + return self.datetime.strftime('%Y-%m-%d_%H%M_%Z') + + def __str__(self) -> str: + return ( + f'{self.__class__.__name__}' + + f'(ts={self.GetTimestampStr()}, ' + + f'tz={self.GetTimezoneAbbr()}, ' + + f'offset={self.GetTimezoneOffsetMin()})' + ) + + @classmethod + def FromComponents( + cls, + year: int, + month: int, + day: int, + hour: int, + minute: int, + second: int, + tz: str, + ) -> 'Datetime': + tzInfo = zoneinfo.ZoneInfo(tz) + + return cls( + datetime=datetime.datetime( + year=year, + month=month, + day=day, + hour=hour, + minute=minute, + second=second, + tzinfo=tzInfo + ) + ) + diff --git a/zyAPI/Due/Due.py b/zyAPI/Due/Due.py new file mode 100644 index 0000000..429cd40 --- /dev/null +++ b/zyAPI/Due/Due.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import pandas + +from .Datetime import Datetime + + +class Due(object): + + def __init__( + self, + dueDate: Datetime, + ) -> None: + super(Due, self).__init__() + + self.dueDate = dueDate + + def Apply2Pd( + self, + df: pandas.DataFrame, + totalColName: str, + ) -> pandas.DataFrame: + raise NotImplementedError('Apply2Pd not implemented') + diff --git a/zyAPI/Due/DueWithLambdaPolicy.py b/zyAPI/Due/DueWithLambdaPolicy.py new file mode 100644 index 0000000..6820818 --- /dev/null +++ b/zyAPI/Due/DueWithLambdaPolicy.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import pandas + +from typing import Callable +from .Datetime import Datetime +from .Due import Due + + +class DueWithLambdaPolicy(Due): + + def __init__( + self, + dueDate: Datetime, + policy: Callable[[float], float] = lambda x: x, + ) -> None: + super(DueWithLambdaPolicy, self).__init__(dueDate=dueDate) + + self.policy = policy + + def Apply2Pd( + self, + df: pandas.DataFrame, + totalColName: str, + destColName: str, + ) -> pandas.DataFrame: + df[destColName] = df[totalColName].apply(self.policy) + return df + diff --git a/zyAPI/Due/__init__.py b/zyAPI/Due/__init__.py new file mode 100644 index 0000000..97aa456 --- /dev/null +++ b/zyAPI/Due/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + diff --git a/zyAPI/Host.py b/zyAPI/Host.py new file mode 100644 index 0000000..552207f --- /dev/null +++ b/zyAPI/Host.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import json +import logging +import time +import requests + +from .Auth.Auth import Auth + +class Host(object): + + LOGGER = logging.getLogger(f'{__name__}') + + def __init__(self, host: str) -> None: + super(Host, self).__init__() + + self.logger = logging.getLogger(f'{__name__}.{self.__class__.__name__}') + + self.host = host + self.session = requests.Session() + self.exportSession = requests.Session() + + def __str__(self) -> str: + return f'Host(host={self.host})' + + def GetHost(self) -> str: + return self.host + + @classmethod + def CheckRespJsonSuccess(cls, resp: dict) -> dict: + if not resp['success']: + err = resp['error'] + errStr = json.dumps(err, indent='\t') + raise RuntimeError(f'API call failed: {errStr}') + + return resp + + def ExportWait( + self, + auth: Auth, + exportDict: dict, + pollInterval: float=1.0, + pollTimes: int=50 + ) -> str: + if not exportDict['success']: + raise RuntimeError('Export failed') + + loc = exportDict['location'] + + headers = {} + auth.AddAuth(headers) + + for _ in range(pollTimes): + resp = self.exportSession.get(loc, headers=headers) + statusDict = resp.json() + + if not statusDict['success']: + statusDictStr = json.dumps(statusDict, indent='\t') + self.logger.error(f'Export status failed:\n{statusDictStr}') + raise RuntimeError('Export status failed') + + if statusDict['state'] == 'PENDING': + self.logger.debug('Export pending') + time.sleep(pollInterval) + elif statusDict['state'] == 'SUCCESS': + self.logger.debug('Export success') + url = statusDict['url'] + return url + else: + state = statusDict['state'] + raise RuntimeError(f'Export failed with state: {state}') + + raise RuntimeError('Status polling exceeded maximum attempts') + diff --git a/zyAPI/Interfaces/Assignment.py b/zyAPI/Interfaces/Assignment.py new file mode 100644 index 0000000..41305fa --- /dev/null +++ b/zyAPI/Interfaces/Assignment.py @@ -0,0 +1,428 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + + +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import numpy +import re +import pandas + +from typing import Callable, List, Tuple +from ..Auth.Auth import Auth +from ..Host import Host +from ..Due import Due +from ..Due import Datetime + + +class Section(object): + + def __init__(self, payload: dict) -> None: + super(Section, self).__init__() + + self.payload = payload + + self.id = self.payload['canonical_section_id'] + self.title = self.payload['title'] + self.totalPts = self.payload['total_points'] + self.incPart = self.payload['include_participations'] + self.incChal = self.payload['include_challenges'] + self.incLabs = self.payload['include_labs'] + + def __str__(self) -> str: + return f'Section(id={self.id}, title={self.title}, totalPts={self.totalPts})' + + def CalcTotalPtsByColInfo(self, colInfo: dict) -> float: + total = 0.0 + if self.incPart: + total += colInfo['pts']['partTotal'] + if self.incChal: + total += colInfo['pts']['chalTotal'] + if self.incLabs: + total += colInfo['pts']['labsTotal'] + return total + + def AssertTotalsPtsWithColInfo(self, colInfo: dict) -> None: + total = self.CalcTotalPtsByColInfo(colInfo) + assert float(self.totalPts) == total, 'Total points mismatch' + + +class Sections(object): + + def __init__(self, payloads: List[dict]) -> None: + super(Sections, self).__init__() + + self.sections = [Section(payload) for payload in payloads] + + def __str__(self) -> str: + return f'Sections({self.GetIdList()})' + + def GetIdList(self) -> List[int]: + return [section.id for section in self.sections] + + def GetTotalPtsBySecId(self, secId: int) -> float: + for section in self.sections: + if section.id == secId: + return section.totalPts + raise ValueError(f'Section id {secId} not found') + + +class Assignment(object): + + def __init__( + self, + host:Host, + auth: Auth, + course: 'Course', + payload: dict, + ) -> None: + super(Assignment, self).__init__() + + self.host = host + self.auth = auth + self.course = course + self.payload = payload + + self.id = self.payload['assignment_id'] + self.creatorId = self.payload['creator_user_id'] + self.title = self.payload['title'] + self.visible = self.payload['visible'] == 1 + self.sections = Sections(self.payload['sections']) + + def __str__(self) -> str: + return f'Assignment(id={self.id}, title={self.title}, visible={self.visible})' + + @classmethod + def ParseReportHeader(cls, headers: List[str]) -> dict: + ''' + Examples: + ``` + [ + 'Total (52)', + 'Participation total (36)', + 'Challenge total (16)', + 'Lab total (0)', + '1.1 - Participation (21)', + '1.2 - Participation (15)', + '1.1 - Challenge (3)', + '1.2 - Challenge (13)', + '1.1 - Lab (0)', + '1.2 - Lab (0)' + ] + ``` + ''' + REGEX_LNAME = r'^\s*[Ll]ast\s+[Nn]ame\s*$' + REGEX_FNAME = r'^\s*[Ff]irst\s+[Nn]ame\s*$' + REGEX_PRI_EMAIL = r'^\s*[Pp]rimary\s+[Ee]mail\s*$' + REGEX_SCH_EMAIL = r'^\s*[Ss]chool\s+[Ee]mail\s*$' + REGEX_TOTAL = r'^\s*[Tt]otal\s*\((\d+)\)\s*$' + REGEX_PART_TOTAL = r'^\s*[Pp]articipation\s+[Tt]otal\s*\((\d+)\)\s*$' + REGEX_CHAL_TOTAL = r'^\s*[Cc]hallenge\s+[Tt]otal\s*\((\d+)\)\s*$' + REGEX_LABS_TOTAL = r'^\s*[Ll]ab\s+[Tt]otal\s*\((\d+)\)\s*$' + REGEX_PART = r'^\s*([0-9.]+)\s*-\s*[Pp]articipation\s*\((\d+)\)\s*$' + REGEX_CHAL = r'^\s*([0-9.]+)\s*-\s*[Cc]hallenge\s*\((\d+)\)\s*$' + REGEX_LABS = r'^\s*([0-9.]+)\s*-\s*[Ll]ab\s*\((\d+)\)\s*$' + + info = { + 'idx': { + 'part': [], + 'chal': [], + 'labs': [], + }, + 'pts': { + 'part': [], + 'chal': [], + 'labs': [], + }, + } + for i in range(len(headers)): + if match := re.match(REGEX_LNAME, headers[i]): + if 'lname' in info['idx']: + raise RuntimeError('Duplicate last name column') + info['idx']['lname'] = i + elif match := re.match(REGEX_FNAME, headers[i]): + if 'fname' in info['idx']: + raise RuntimeError('Duplicate first name column') + info['idx']['fname'] = i + elif match := re.match(REGEX_SCH_EMAIL, headers[i]): + if 'schEmail' in info['idx']: + raise RuntimeError('Duplicate school email column') + info['idx']['schEmail'] = i + elif match := re.match(REGEX_PRI_EMAIL, headers[i]): + if 'priEmail' in info['idx']: + raise RuntimeError('Duplicate primary email column') + info['idx']['priEmail'] = i + + elif match := re.match(REGEX_TOTAL, headers[i]): + if 'total' in info['idx']: + raise RuntimeError('Duplicate total column') + info['idx']['total'] = i + info['pts']['total'] = float(match.group(1)) + + elif match := re.match(REGEX_PART_TOTAL, headers[i]): + if 'partTotal' in info['idx']: + raise RuntimeError('Duplicate participation total column') + info['idx']['partTotal'] = i + info['pts']['partTotal'] = float(match.group(1)) + elif match := re.match(REGEX_CHAL_TOTAL, headers[i]): + if 'chalTotal' in info['idx']: + raise RuntimeError('Duplicate challenge total column') + info['idx']['chalTotal'] = i + info['pts']['chalTotal'] = float(match.group(1)) + elif match := re.match(REGEX_LABS_TOTAL, headers[i]): + if 'labsTotal' in info['idx']: + raise RuntimeError('Duplicate lab total column') + info['idx']['labsTotal'] = i + info['pts']['labsTotal'] = float(match.group(1)) + + elif match := re.match(REGEX_PART, headers[i]): + info['idx']['part'].append(i) + info['pts']['part'].append(float(match.group(2))) + elif match := re.match(REGEX_CHAL, headers[i]): + info['idx']['chal'].append(i) + info['pts']['chal'].append(float(match.group(2))) + elif match := re.match(REGEX_LABS, headers[i]): + info['idx']['labs'].append(i) + info['pts']['labs'].append(float(match.group(2))) + + # print(info) + + # validate + if 'lname' not in info['idx']: + raise RuntimeError('Missing last name column') + if 'fname' not in info['idx']: + raise RuntimeError('Missing first name column') + if 'priEmail' not in info['idx']: + raise RuntimeError('Missing primary email column') + if 'schEmail' not in info['idx']: + raise RuntimeError('Missing school email column') + + if 'total' not in info['idx']: + raise RuntimeError('Missing total column') + if 'partTotal' not in info['idx']: + raise RuntimeError('Missing participation total column') + if 'chalTotal' not in info['idx']: + raise RuntimeError('Missing challenge total column') + if 'labsTotal' not in info['idx']: + raise RuntimeError('Missing lab total column') + + if ( + ( + info['pts']['partTotal'] + + info['pts']['chalTotal'] + + info['pts']['labsTotal'] + ) != + info['pts']['total'] + ): + raise RuntimeError('Total mismatch') + + if sum(info['pts']['part']) != info['pts']['partTotal']: + raise RuntimeError('Participation total mismatch') + if sum(info['pts']['chal']) != info['pts']['chalTotal']: + raise RuntimeError('Challenge total mismatch') + if sum(info['pts']['labs']) != info['pts']['labsTotal']: + raise RuntimeError('Lab total mismatch') + + return info + + def _CourseExportReportByDate( + self, + date: Datetime.Datetime, + secIds: List[int], + includeTimeSpent: bool=False, + ) -> Tuple[str, pandas.DataFrame]: + return self.course.ExportReportByDate( + date=date, + secIds=secIds, + includeTimeSpent=includeTimeSpent, + ) + + def ExportReportByDate( + self, + date: Datetime.Datetime, + includeTimeSpent: bool=False, + ) -> Tuple[str, pandas.DataFrame]: + dfs = {} + for sec in self.sections.sections: + secId = sec.id + + filename, df = self._CourseExportReportByDate( + date=date, + secIds=[secId], + includeTimeSpent=includeTimeSpent, + ) + + # rename columns and drop unwanted columns + unwantedCols = [] + colInfo = self.ParseReportHeader(list(df.columns)) + for i in range(len(df.columns)): + # names, emails + if i == colInfo['idx']['lname']: + df.columns.values[i] = 'last_name' + elif i == colInfo['idx']['fname']: + df.columns.values[i] = 'first_name' + elif i == colInfo['idx']['priEmail']: + df.columns.values[i] = 'primary_email' + elif i == colInfo['idx']['schEmail']: + df.columns.values[i] = 'school_email' + + # total points + elif i == colInfo['idx']['total']: + df.columns.values[i] = f'{secId}' + elif (i == colInfo['idx']['partTotal']) and sec.incPart: + df.columns.values[i] = f'{secId}.part' + elif (i == colInfo['idx']['chalTotal']) and sec.incChal: + df.columns.values[i] = f'{secId}.chal' + elif (i == colInfo['idx']['labsTotal']) and sec.incLabs: + df.columns.values[i] = f'{secId}.labs' + else: + # give unwanted columns a unique name + df.columns.values[i] = f'unwanted_{i}' + unwantedCols.append(i) + + df.drop(columns=df.columns[unwantedCols], inplace=True) + + # fillna with 0.0 + df[f'{secId}'] = df[f'{secId}'].fillna(0.0) + if sec.incPart: + df[f'{secId}.part'] = df[f'{secId}.part'].fillna(0.0) + if sec.incChal: + df[f'{secId}.chal'] = df[f'{secId}.chal'].fillna(0.0) + if sec.incLabs: + df[f'{secId}.labs'] = df[f'{secId}.labs'].fillna(0.0) + + # make primary_email as the key column + df.set_index('primary_email', inplace=True) + + # validate the total points + sec.AssertTotalsPtsWithColInfo(colInfo) + + # calculate points by col_values(i.e., percent) x total points + df[f'{secId}'] = 0.0 + if sec.incPart: + df[f'{secId}.part'] *= colInfo['pts']['partTotal'] + df[f'{secId}.part'] /= 100 + # add to total + df[f'{secId}'] += df[f'{secId}.part'] + if sec.incChal: + df[f'{secId}.chal'] *= colInfo['pts']['chalTotal'] + df[f'{secId}.chal'] /= 100 + # add to total + df[f'{secId}'] += df[f'{secId}.chal'] + if sec.incLabs: + df[f'{secId}.labs'] *= colInfo['pts']['labsTotal'] + df[f'{secId}.labs'] /= 100 + # add to total + df[f'{secId}'] += df[f'{secId}.labs'] + + # save the dataframe + dfs[secId] = df + + # merge the dataframes + if len(dfs) == 0: + raise RuntimeError('No dataframes') + + dfsecIds = list(dfs.keys()) + df = dfs[dfsecIds[0]] + dfRowCnt = len(df) + + for i in range(1, len(dfsecIds)): + df = pandas.merge( + df, dfs[dfsecIds[i]], + how='inner', + on='primary_email', + suffixes=('', f'_{dfsecIds[i]}'), + ) + + # make sure merged dataframe doesn't lost any rows + if len(df) < dfRowCnt: + raise RuntimeError('Merged dataframe lost rows') + dfRowCnt = len(df) + + # remove duplicate columns + df.drop(columns=[f'last_name_{dfsecIds[i]}'], inplace=True) + df.drop(columns=[f'first_name_{dfsecIds[i]}'], inplace=True) + df.drop(columns=[f'school_email_{dfsecIds[i]}'], inplace=True) + + # create total column + df['total'] = 0.0 + df['total_percent'] = 0.0 + asgTotal = 0.0 + for secId in dfsecIds: + df['total'] += df[f'{secId}'] + asgTotal += self.sections.GetTotalPtsBySecId(secId) + + df['total_percent'] += (df['total'] * 100) / asgTotal + + return filename, df + + def ExportReportWithDue( + self, + due: Due.Due, + includeTimeSpent: bool=False, + ) -> Tuple[str, pandas.DataFrame]: + filename, df = self.ExportReportByDate( + date=due.dueDate, + includeTimeSpent=includeTimeSpent + ) + + due.Apply2Pd( + df=df, + totalColName='total', + destColName='total_due', + ) + + due.Apply2Pd( + df=df, + totalColName='total_percent', + destColName='total_percent_due', + ) + + return filename, df + + def ExportReportWithDues( + self, + dues: List[Due.Due], + includeTimeSpent: bool=False, + mergeOps: Callable = numpy.maximum, + ) -> Tuple[str, pandas.DataFrame]: + if len(dues) == 0: + raise ValueError('No dues specified') + + filename, df = self.ExportReportWithDue( + due=dues[0], + includeTimeSpent=includeTimeSpent, + ) + + for due in dues: + _, dfNext = self.ExportReportWithDue( + due=due, + includeTimeSpent=includeTimeSpent, + ) + for i in range(len(df.columns)): + colName = df.columns.values[i] + for sec in self.sections.sections: + if f'{sec.id}' in colName: + df[colName] = mergeOps(df[colName], dfNext[colName]) + + if 'total' in colName: + df[colName] = mergeOps(df[colName], dfNext[colName]) + #print(df[colName].keys()) + + return filename, df + diff --git a/zyAPI/Interfaces/Course.py b/zyAPI/Interfaces/Course.py new file mode 100644 index 0000000..0b6a08b --- /dev/null +++ b/zyAPI/Interfaces/Course.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import io +import json +import logging +import pandas +import os + +from typing import List, Tuple, Union +from ..Auth.Auth import Auth +from ..Due import Datetime +from ..Host import Host +from .Assignment import Assignment + +class Course(object): + + def __init__( + self, + host:Host, + auth: Auth, + dashboard: 'Dashboard', + payload: dict + ) -> None: + super(Course, self).__init__() + + self.logger = logging.getLogger(f'{__name__}.{self.__class__.__name__}') + + self.host = host + self.auth = auth + self.dashboard = dashboard + self.payload = payload + + self.id = self.payload['zybook_id'] + self.code = self.payload['zybook_code'] + self.title = self.payload['title'] + + def __str__(self) -> str: + return f'Course(id={self.id}, code={self.code}, title={self.title})' + + def GetRoster( + self, + roles: List[str]=['Instructor','TA','Student','Temporary','Dropped'] + ) -> dict: + path = f'/v1/zybook/{self.code}/roster' + url = f'https://{self.host.GetHost()}{path}' + + params = { + 'zybook_roles': json.dumps(roles,separators=(',', ':')), + } + headers = {} + + self.auth.AddAuth(headers) + + response = self.host.session.get(url, headers=headers, params=params) + + return self.host.CheckRespJsonSuccess(response.json()) + + def GetAssignments(self) -> dict: + path = f'/v1/zybook/{self.code}/assignments' + url = f'https://{self.host.GetHost()}{path}' + + headers = {} + + self.auth.AddAuth(headers) + + response = self.host.session.get(url, headers=headers) + + return self.host.CheckRespJsonSuccess(response.json()) + + def OpenAssignment( + self, + assignmentID: Union[int, None]=None, + titleKeyword: Union[str, None]=None, + ) -> Assignment: + if ( + assignmentID is not None and + titleKeyword is not None + ): + raise ValueError( + 'Only one of the search parameters can be specified' + ) + + assignmentList = self.GetAssignments() + + payload = None + for assignment in assignmentList['assignments']: + if assignmentID is not None: + if assignment['assignment_id'] == assignmentID: + if payload is not None: + raise ValueError('Assignment ID not unique') + payload = assignment + + elif titleKeyword is not None: + if titleKeyword in assignment['title']: + if payload is not None: + raise ValueError('Title keyword not unique') + payload = assignment + + if payload is None: + raise ValueError('Assignment not found') + else: + return Assignment( + host=self.host, + auth=self.auth, + course=self, + payload=payload + ) + + def ExportReportByDate( + self, + date: Datetime.Datetime, + secIds: List[int], + includeTimeSpent: bool=False, + ) -> Tuple[str, pandas.DataFrame]: + path = f'/v1/zybook/{self.code}/activities/export' + url = f'https://{self.host.GetHost()}{path}' + + headers = {} + self.auth.AddAuth(headers) + + # pull the report + params = { + 'time_zone_abbreviation': date.GetTimezoneAbbr(), + 'time_zone_offset': date.GetZyTimezoneOffsetMin(), + 'end_date': date.GetUtcTimestampStr(), + 'sections': json.dumps(secIds,separators=(',', ':')), + 'include_time_spent': includeTimeSpent, + 'combine_activities': False, + 'assignment_id': '', + } + resp = self.host.session.get(url, params=params, headers=headers) + respJson = self.host.CheckRespJsonSuccess(resp.json()) + + # wait for the report to be ready + csvUrl = self.host.ExportWait(auth=self.auth, exportDict=respJson) + + # download the report + csvResp = self.host.session.get(csvUrl, headers=headers) + + csvStr = csvResp.text + filename = os.path.basename(csvUrl) + + self.logger.debug(f'Exported report: {filename}') + expectedTimeSuffix = date.GetReportNameSuffix() + if expectedTimeSuffix not in filename: + raise ValueError(f'Expected time suffix not found in filename: {expectedTimeSuffix}') + + df = pandas.read_csv(io.StringIO(csvStr)) + + return filename, df + diff --git a/zyAPI/Interfaces/Dashboard.py b/zyAPI/Interfaces/Dashboard.py new file mode 100644 index 0000000..6cb2dcc --- /dev/null +++ b/zyAPI/Interfaces/Dashboard.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +from typing import Union +from ..Auth.Auth import Auth +from ..Host import Host +from .Course import Course + + +class Dashboard(object): + + def __init__(self, host:Host, auth: Auth, uid: int) -> None: + super(Dashboard, self).__init__() + + self.host = host + self.auth = auth + self.uid = uid + + def __str__(self) -> str: + return f'Dashboard(uid={self.uid})' + + def GetUserInfo(self) -> dict: + path = f'/v1/user/{self.uid}' + url = f'https://{self.host.GetHost()}{path}' + + headers = {} + + self.auth.AddAuth(headers) + + response = self.host.session.get(url, headers=headers) + + return response.json() + + def GetCourseList(self) -> dict: + path = f'/v1/user/{self.uid}/items' + url = f'https://{self.host.GetHost()}{path}' + + headers = {} + + self.auth.AddAuth(headers) + + response = self.host.session.get(url, headers=headers) + + return response.json() + + def OpenCourse( + self, + courseID: Union[int, None]=None, + courseCode: Union[str, None]=None, + titleKeyword: Union[str, None]=None, + ) -> Course: + if ( + courseID is not None and + courseCode is not None and + titleKeyword is not None + ): + raise ValueError( + 'Only one of the search parameters can be specified' + ) + + courseList = self.host.CheckRespJsonSuccess(self.GetCourseList()) + + payload = None + for course in courseList['items']['zybooks']: + if courseID is not None: + if course['zybook_id'] == courseID: + if payload is not None: + raise ValueError('Course ID not unique') + payload = course + + elif courseCode is not None: + if course['zybook_code'] == courseCode: + if payload is not None: + raise ValueError('Course code not unique') + payload = course + + elif titleKeyword is not None: + if titleKeyword in course['title']: + if payload is not None: + raise ValueError('Title keyword not unique') + payload = course + + if payload is None: + raise ValueError('Course not found') + else: + return Course( + host=self.host, + auth=self.auth, + dashboard=self, + payload=payload + ) diff --git a/zyAPI/Interfaces/__init__.py b/zyAPI/Interfaces/__init__.py new file mode 100644 index 0000000..97aa456 --- /dev/null +++ b/zyAPI/Interfaces/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + diff --git a/zyAPI/Utils.py b/zyAPI/Utils.py new file mode 100644 index 0000000..ef587f2 --- /dev/null +++ b/zyAPI/Utils.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import pandas +import re + + +class Report(object): + + def __init__(self) -> None: + super(Report, self).__init__() + + @classmethod + def MergeEmailCols( + cls, + df: pandas.DataFrame, + preferredEmailFormat: str + ) -> None: + ''' + Merge primary_email and school_email columns into an email column, by + checking if the primary_email is preferredEmailFormat, + if not, check if the school_email is preferredEmailFormat, + otherwise, keep the primary_email. + ''' + + # iterate through the rows + for index, row in df.iterrows(): + # check if the index(primary_email) is preferredEmailFormat + if re.search(preferredEmailFormat, index): + # if yes, keep index(primary_email) and continue + df.at[index, 'email'] = index + # check if the school_email is NaN + elif pandas.isna(row['school_email']): + # if school_email is NaN, keep index(primary_email) and continue + df.at[index, 'email'] = index + # check if the school_email is preferredEmailFormat + elif re.search(preferredEmailFormat, row['school_email']): + # if yes, replace index(primary_email) with school_email + df.at[index, 'email'] = row['school_email'] + + # drop school_email column + df.drop(columns=['school_email'], inplace=True) + + # drop index(primary_email) column + df.reset_index(drop=True, inplace=True) + + # make the email column the index + df.set_index('email', inplace=True) + + # make sure there is no duplicate email + duplicateEmails = df.index.duplicated() + if True in duplicateEmails: + # get the duplicate emails + duplicateEmails = df.loc[duplicateEmails, 'email'] + # raise an error + raise ValueError(f'Duplicate email after merging the columns: {duplicateEmails}') + + @classmethod + def ReplaceEmailsByMap( + cls, + df: pandas.DataFrame, + emailMap: dict, + ) -> None: + ''' + Replace emails in the dataframe by the emailMap. + ''' + + df.reset_index(drop=False, inplace=True) + df.replace({'email': emailMap}, inplace=True) + df.set_index('email', inplace=True) + + # make sure there is no duplicate email + duplicateEmails = df.index.duplicated() + if True in duplicateEmails: + # get the duplicate emails + duplicateEmails = df.loc[duplicateEmails, 'email'] + raise ValueError(f'Duplicate email after replacing emails by map: {duplicateEmails}') + + @classmethod + def CheckEmailsFormat( + cls, + df: pandas.DataFrame, + emailFormat: str, + ) -> None: + ''' + Check if the emails in the dataframe match the emailFormat. + ''' + invliadEmails = [] + for index, row in df.iterrows(): + if not re.search(emailFormat, index): + invliadEmails.append({ + 'email': index, + 'name': f'{row["first_name"]} {row["last_name"]}' + }) + + if len(invliadEmails) > 0: + raise ValueError(f'Invalid emails: {invliadEmails}') + diff --git a/zyAPI/_Meta.py b/zyAPI/_Meta.py new file mode 100644 index 0000000..c472a98 --- /dev/null +++ b/zyAPI/_Meta.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + + +__version__ = '0.1.0' + +PKG_AUTHOR = 'Haofan Zheng, Languages Systems and Data Lab, UC Santa Cruz' +PKG_NAME = 'zyAPI' +PKG_DESCRIPTION = 'Some useful API requesters for zyBooks\'s API' +PKG_LICENSE = 'MIT' diff --git a/zyAPI/__init__.py b/zyAPI/__init__.py new file mode 100644 index 0000000..97aa456 --- /dev/null +++ b/zyAPI/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + diff --git a/zyAPI/__main__.py b/zyAPI/__main__.py new file mode 100644 index 0000000..5ff153b --- /dev/null +++ b/zyAPI/__main__.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# -*- coding:utf-8 -*- +### +# Copyright (c) 2024 Haofan Zheng +# Use of this source code is governed by an MIT-style +# license that can be found in the LICENSE file or at +# https://opensource.org/licenses/MIT. +### + + +import argparse + +from ._Meta import __version__ + + +def main(): + parser = argparse.ArgumentParser( + description='zyAPI - A Python package for zyBooks\'s API' + ) + + parser.add_argument( + '--version', + action='version', + version=f'{__version__}' + ) + + args = parser.parse_args() + + +if __name__ == '__main__': + main()