From 50517f17448d7d893f50b48a00f617d7a064d498 Mon Sep 17 00:00:00 2001 From: Krzysztof Madejski Date: Tue, 5 Nov 2019 12:13:17 +0100 Subject: [PATCH] Progress veryifying document or task WIP #138 --- .../migrations/0004_auto_20191105_1004.py | 24 ++++++++++++ moonsheep/models.py | 16 +++++++- moonsheep/settings.py | 8 +++- moonsheep/statistics.py | 13 +++++++ moonsheep/tasks.py | 39 +++++++++++++------ moonsheep/templatetags/moonsheep.py | 7 +++- moonsheep/views.py | 2 +- 7 files changed, 91 insertions(+), 18 deletions(-) create mode 100644 moonsheep/migrations/0004_auto_20191105_1004.py create mode 100644 moonsheep/statistics.py diff --git a/moonsheep/migrations/0004_auto_20191105_1004.py b/moonsheep/migrations/0004_auto_20191105_1004.py new file mode 100644 index 0000000..649816d --- /dev/null +++ b/moonsheep/migrations/0004_auto_20191105_1004.py @@ -0,0 +1,24 @@ +# Generated by Django 2.2.5 on 2019-11-05 10:04 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('moonsheep', '0003_auto_20190821_1501'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='own_progress', + field=models.DecimalField(decimal_places=3, default=0, max_digits=6, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(0)]), + ), + migrations.AddField( + model_name='task', + name='total_progress', + field=models.DecimalField(decimal_places=3, default=0, max_digits=6, validators=[django.core.validators.MaxValueValidator(100), django.core.validators.MinValueValidator(0)]), + ), + ] diff --git a/moonsheep/models.py b/moonsheep/models.py index fed13cb..21a80e3 100644 --- a/moonsheep/models.py +++ b/moonsheep/models.py @@ -54,17 +54,29 @@ class Task(models.Model): """ # TODO issue with circular imports; resolve it otherwise, add choices dynamically or drop it # from .registry import TASK_TYPES - type = models.CharField(verbose_name=_("Type"), max_length=255) #, choices=[(t, t) for t in TASK_TYPES]) + type = models.CharField(verbose_name=_("Type"), max_length=255) # , choices=[(t, t) for t in TASK_TYPES]) """Full reference (with module) to task class name""" params = JSONField(blank=True) """Params specifying the task, that will be passed to user""" - # TODO count priority + interface + parent_id = models.ForeignKey('Task', models.CASCADE, null=True) + """Set if this task is a child of another""" + + doc_id = models.IntegerField() + """Pointing to document_id being processed by this task""" + + # TODO count priority + interface https://github.com/themoonsheep/moonsheep/issues/50 priority = models.DecimalField(decimal_places=2, max_digits=3, default=1.0, validators=[validators.MaxValueValidator(1.0), validators.MinValueValidator(0.0)], ) """Priority of the task, set manually or computed by defined functionD from other fields. Scale: 0.0 - 1.0""" + own_progress = models.DecimalField(decimal_places=3, max_digits=6, default=0, + validators=[validators.MaxValueValidator(100), validators.MinValueValidator(0)]) + + total_progress = models.DecimalField(decimal_places=3, max_digits=6, default=0, + validators=[validators.MaxValueValidator(100), + validators.MinValueValidator(0)]) # States OPEN = 'open' DIRTY = 'dirty' diff --git a/moonsheep/settings.py b/moonsheep/settings.py index 66033f9..d9abcbb 100644 --- a/moonsheep/settings.py +++ b/moonsheep/settings.py @@ -1,7 +1,13 @@ # TODO A tested solution for importing & overriding settings: # https://github.com/encode/django-rest-framework/blob/master/rest_framework/settings.py +# We go with more basic solution +# 1) isting here default settings +# 2) importing them in a project: from moonsheep.settings import * # NOQA +# 3) updating them if needed: MOONSHEEP.update({}) + MOONSHEEP = { - 'DEV_ROTATE_TASKS': False + 'DEV_ROTATE_TASKS': False, + 'MIN_ENTRIES_TO_CROSSCHECK': 3, } REST_FRAMEWORK = { diff --git a/moonsheep/statistics.py b/moonsheep/statistics.py new file mode 100644 index 0000000..aab18de --- /dev/null +++ b/moonsheep/statistics.py @@ -0,0 +1,13 @@ +from moonsheep import models + + +def update_parents_progress(task: models.Task): + if task.parent_id: + pass + # TODO update parent total progress #138 + # ( average(subtasks progress) * average_number_of_subtasks(configured in task) + own_progress ) / (average_number_of_subtasks(configured in task) + 1) + # own_progress of parent == 1 + else: + # it must be a doc-descendant task + # TODO update doc progress #138 + pass diff --git a/moonsheep/tasks.py b/moonsheep/tasks.py index 29a6e25..ccda315 100644 --- a/moonsheep/tasks.py +++ b/moonsheep/tasks.py @@ -1,10 +1,13 @@ import logging +import math from typing import List, Type from django.db import transaction from django.utils.decorators import classproperty +from moonsheep import statistics from moonsheep.models import Task, Entry +from moonsheep.settings import MOONSHEEP from .mapper import klass_from_name from .verifiers import MIN_CONFIDENCE, DEFAULT_DICT_VERIFIER from . import registry @@ -12,6 +15,7 @@ logger = logging.getLogger(__name__) +# TODO rename to TaskType? add ABC class? class AbstractTask(object): N_ANSWERS = 1 @@ -69,12 +73,25 @@ def verify_and_save(self, entries: List[Entry]) -> bool: # save verified data with transaction.atomic(): self.save_verified_data(crosschecked) + # create new tasks self.after_save(crosschecked) + # update progress & state + self.instance.own_progress = 100 + statistics.update_parents_progress(self.instance) + + self.instance.state = Task.CROSSCHECKED + self.instance.save() + return True else: - # TODO: do something here + # update progress + self.instance.own_progress = 95 * (1 - math.exp(-2 / MOONSHEEP['MIN_ENTRIES_TO_CROSSCHECK'] * len(entries))) + statistics.update_parents_progress(self.instance) + + self.instance.save() + return False # TODO record somewhere on how many entries the crosscheck was done, update values if new crosscheck comes with higher rank @@ -110,22 +127,20 @@ def after_save(self, verified_data): pass @classmethod - def create(cls, params: dict) -> Task: + def create(cls, params: dict, parent: Task) -> Task: """ - Helper method for creating a new task. + Helper method for creating a new task of given type. - :param task_type: Type of task you want to create :param params: Params to initialize the task + :param parent: Parent task :return: created task """ - t = Task(type=cls.name, params=params) - t.save() - - return t + return Task.objects.create(type=cls.name, params=params, parent_id=parent.id, doc_id=parent.doc_id) + # TODO change convention @staticmethod - def create_task_instance(task: Task): + def create_task_instance(task: Task) -> 'AbstractTask': """ Create relevant task instance. @@ -138,10 +153,10 @@ def create_task_instance(task: Task): # TODO call it from somewhere @staticmethod - def verify_task(task_or_id): # TODO, simplify to one type + def verify_task(task_or_id): # TODO, simplify to one type if isinstance(task_or_id, int): task_or_id = Task.objects.get(task_or_id) - if not isinstance(task_or_id, Task): + if not isinstance(task_or_id, Task): raise ValueError("task must be task_id (int) or a Task") # TODO find better name convention @@ -172,4 +187,4 @@ def _task_wrapper(task_class): return task_class - return _task_wrapper \ No newline at end of file + return _task_wrapper diff --git a/moonsheep/templatetags/moonsheep.py b/moonsheep/templatetags/moonsheep.py index d7dc6ee..f8ec343 100644 --- a/moonsheep/templatetags/moonsheep.py +++ b/moonsheep/templatetags/moonsheep.py @@ -6,6 +6,7 @@ import django.db.models import urllib.parse +from moonsheep.models import Task from moonsheep.tasks import AbstractTask from moonsheep.settings import MOONSHEEP @@ -30,14 +31,16 @@ def document_change_url(instance): @register.simple_tag def progress_of(document_or_task): if isinstance(document_or_task, django.db.models.Model): + # document doc = document_or_task tasks = MOONSHEEP['DOCUMENT_INITIAL_TASKS'] # TODO #13 else: - task = document_or_task + task: Task = document_or_task + return task.own_progress # TODO - # TODO #13 + # TODO #13 #138 return 81 diff --git a/moonsheep/views.py b/moonsheep/views.py index afe9e08..6ade382 100644 --- a/moonsheep/views.py +++ b/moonsheep/views.py @@ -250,7 +250,7 @@ def _save_entry(self, task_id, data) -> None: # Do the crosscheck if we have enough entries entries = Entry.objects.filter(task_id=task_id) - if entries.count() > 3: # TODO setting how many entries do we want for crosscheck + if entries.count() >= MOONSHEEP['MIN_ENTRIES_TO_CROSSCHECK']: self.task_type.verify_and_save(list(entries)) def _get_user_ip(self):