feat: 284 and 285 - vin decoding (#289)

* feat: 284 and 285 - vin decoding

* update migration

* add db table comments

django/api/decoder_constants.py

@@ -0,0 +1,32 @@
+import os
+from enum import Enum
+from functools import partial
+from api.models.decoded_vin_record import VpicDecodedVinRecord, VinpowerDecodedVinRecord
+from workers.external_apis.vpic import batch_decode as vpic_batch_decode
+from workers.external_apis.vinpower import batch_decode as vinpower_batch_decode
+
+
+class VPIC(Enum):
+    NAME = "vpic"
+    CURRENT_DECODE_SUCCESSFUL = "vpic_current_decode_successful"
+    NUMBER_OF_CURRENT_DECODE_ATTEMPTS = "vpic_number_of_current_decode_attempts"
+    MODEL = VpicDecodedVinRecord
+    BATCH_DECODER = partial(vpic_batch_decode)
+
+
+class VINPOWER(Enum):
+    NAME = "vinpower"
+    CURRENT_DECODE_SUCCESSFUL = "vinpower_current_decode_successful"
+    NUMBER_OF_CURRENT_DECODE_ATTEMPTS = "vinpower_number_of_current_decode_attempts"
+    MODEL = VinpowerDecodedVinRecord
+    BATCH_DECODER = partial(vinpower_batch_decode)
+
+
+SERVICES = [VPIC, VINPOWER]
+
+
+def get_service(service_name):
+    for service in SERVICES:
+        if service.NAME.value == service_name:
+            return service
+    return None

django/api/logging_filters.py

@@ -0,0 +1,8 @@
+import logging
+
+class HealthcheckFilter(logging.Filter):
+     def filter(self, record):
+         msg = record.getMessage()
+         if "GET /api/healthcheck HTTP/1.1" in msg:
+              return False
+         return True

django/api/management/commands/create_app_user_and_token.py

@@ -0,0 +1,21 @@
+from django.core.management.base import BaseCommand, CommandError
+from api.models.app_user import AppUser, AppToken
+from django.conf import settings
+from django.db import transaction
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("app_name", type=str)
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        app_name = options["app_name"]
+
+        try:
+            app_user = AppUser.objects.create(app_name=app_name)
+            token = AppToken.objects.create(user=app_user)
+        except Exception:
+            raise CommandError("Error generating user and token")
+
+        self.stdout.write("Generated token {} for app {}".format(token.key, app_name))

django/api/management/commands/reset_app_token.py

@@ -0,0 +1,24 @@
+from django.core.management.base import BaseCommand, CommandError
+from api.models.app_user import AppUser, AppToken
+from django.conf import settings
+from django.db import transaction
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument("app_name", type=str)
+
+    @transaction.atomic
+    def handle(self, *args, **options):
+        app_name = options["app_name"]
+
+        try:
+            app_user = AppUser.objects.get(app_name=app_name)
+            AppToken.objects.get(user=app_user).delete()
+            new_token = AppToken.objects.create(user=app_user)
+        except Exception:
+            raise CommandError("Error resetting token")
+
+        self.stdout.write(
+            "Generated new token {} for app {}".format(new_token.key, app_name)
+        )

django/api/migrations/0023_auto_20240514_1721.py

@@ -0,0 +1,116 @@
+# Generated by Django 3.2.25 on 2024-05-14 17:21
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0022_auto_20240503_1823'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='AppToken',
+            fields=[
+                ('key', models.CharField(max_length=40, primary_key=True, serialize=False, verbose_name='Key')),
+                ('created', models.DateTimeField(auto_now_add=True, verbose_name='Created')),
+            ],
+            options={
+                'db_table': 'app_token',
+            },
+        ),
+        migrations.CreateModel(
+            name='AppUser',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)),
+                ('create_user', models.CharField(default='SYSTEM', max_length=130)),
+                ('update_timestamp', models.DateTimeField(auto_now=True, null=True)),
+                ('update_user', models.CharField(max_length=130, null=True)),
+                ('is_active', models.BooleanField(default=True)),
+                ('app_name', models.CharField(max_length=100, unique=True)),
+            ],
+            options={
+                'db_table': 'app_user',
+            },
+        ),
+        migrations.CreateModel(
+            name='UploadedVinRecord',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)),
+                ('create_user', models.CharField(default='SYSTEM', max_length=130)),
+                ('update_timestamp', models.DateTimeField(auto_now=True, null=True)),
+                ('update_user', models.CharField(max_length=130, null=True)),
+                ('vin', models.CharField(max_length=17)),
+                ('postal_code', models.CharField(blank=True, max_length=7, null=True)),
+                ('data', models.JSONField()),
+                ('vpic_current_decode_successful', models.BooleanField(default=False)),
+                ('vpic_number_of_current_decode_attempts', models.IntegerField(default=0)),
+                ('vinpower_current_decode_successful', models.BooleanField(default=False)),
+                ('vinpower_number_of_current_decode_attempts', models.IntegerField(default=0)),
+            ],
+            options={
+                'db_table': 'uploaded_vin_record',
+            },
+        ),
+        migrations.CreateModel(
+            name='UploadedVinsFile',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)),
+                ('create_user', models.CharField(default='SYSTEM', max_length=130)),
+                ('update_timestamp', models.DateTimeField(auto_now=True, null=True)),
+                ('update_user', models.CharField(max_length=130, null=True)),
+                ('filename', models.CharField(max_length=32, unique=True)),
+                ('chunk_size', models.IntegerField(default=25000)),
+                ('chunks_per_run', models.IntegerField(default=4)),
+                ('start_index', models.IntegerField(default=0)),
+                ('processed', models.BooleanField(default=False)),
+            ],
+            options={
+                'db_table': 'uploaded_vins_file',
+            },
+        ),
+        migrations.CreateModel(
+            name='VinpowerDecodedVinRecord',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)),
+                ('create_user', models.CharField(default='SYSTEM', max_length=130)),
+                ('update_timestamp', models.DateTimeField(auto_now=True, null=True)),
+                ('update_user', models.CharField(max_length=130, null=True)),
+                ('vin', models.CharField(max_length=17, unique=True)),
+                ('data', models.JSONField()),
+            ],
+            options={
+                'db_table': 'vinpower_decoded_vin_record',
+            },
+        ),
+        migrations.CreateModel(
+            name='VpicDecodedVinRecord',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('create_timestamp', models.DateTimeField(auto_now_add=True, null=True)),
+                ('create_user', models.CharField(default='SYSTEM', max_length=130)),
+                ('update_timestamp', models.DateTimeField(auto_now=True, null=True)),
+                ('update_user', models.CharField(max_length=130, null=True)),
+                ('vin', models.CharField(max_length=17, unique=True)),
+                ('data', models.JSONField()),
+            ],
+            options={
+                'db_table': 'vpic_decoded_vin_record',
+            },
+        ),
+        migrations.AddConstraint(
+            model_name='uploadedvinrecord',
+            constraint=models.UniqueConstraint(fields=('vin', 'postal_code'), name='unique_vin_postal_code'),
+        ),
+        migrations.AddField(
+            model_name='apptoken',
+            name='user',
+            field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='auth_token', to='api.appuser', verbose_name='User'),
+        ),
+    ]

django/api/models/__init__.py

@@ -21,3 +21,7 @@
 from . import user
 from . import permission
 from . import user_permission
+from . import app_user
+from . import uploaded_vins_file
+from . import uploaded_vin_record
+from . import decoded_vin_record

django/api/models/app_user.py

@@ -0,0 +1,33 @@
+from django.db import models
+from auditable.models import Auditable
+from rest_framework.authtoken.models import Token
+from django.utils.translation import gettext_lazy as _
+
+
+class AppUser(Auditable):
+    is_active = models.BooleanField(default=True)
+
+    app_name = models.CharField(max_length=100, unique=True)
+
+    @property
+    def is_authenticated(self):
+        return True
+
+    class Meta:
+        db_table = "app_user"
+
+    db_table_comment = "represents an external application that integrates this app via API"
+
+
+class AppToken(Token):
+    user = models.OneToOneField(
+        AppUser,
+        related_name="auth_token",
+        on_delete=models.CASCADE,
+        verbose_name=_("User"),
+    )
+
+    class Meta:
+        db_table = "app_token"
+
+    db_table_comment = "the token of an external application that integrates this app via API"

django/api/models/decoded_vin_record.py

@@ -0,0 +1,25 @@
+from django.db import models
+from auditable.models import Auditable
+
+
+class DecodedVinRecord(Auditable):
+    vin = models.CharField(max_length=17, unique=True)
+
+    data = models.JSONField()
+
+    class Meta:
+        abstract = True
+
+
+class VpicDecodedVinRecord(DecodedVinRecord):
+    class Meta:
+        db_table = "vpic_decoded_vin_record"
+
+    db_table_comment = "contains vpic-decoded VIN information"
+
+
+class VinpowerDecodedVinRecord(DecodedVinRecord):
+    class Meta:
+        db_table = "vinpower_decoded_vin_record"
+
+    db_table_comment = "contains vinpower-decoded VIN information"

django/api/models/uploaded_vin_record.py

@@ -0,0 +1,28 @@
+from django.db import models
+from auditable.models import Auditable
+
+
+class UploadedVinRecord(Auditable):
+    vin = models.CharField(max_length=17)
+
+    postal_code = models.CharField(max_length=7, null=True, blank=True)
+
+    data = models.JSONField()
+
+    vpic_current_decode_successful = models.BooleanField(default=False)
+
+    vpic_number_of_current_decode_attempts = models.IntegerField(default=0)
+
+    vinpower_current_decode_successful = models.BooleanField(default=False)
+
+    vinpower_number_of_current_decode_attempts = models.IntegerField(default=0)
+
+    class Meta:
+        db_table = "uploaded_vin_record"
+        constraints = [
+            models.UniqueConstraint(
+                fields=["vin", "postal_code"], name="unique_vin_postal_code"
+            )
+        ]
+
+    db_table_comment = "represents an uploaded VIN, and associated information"

django/api/models/uploaded_vins_file.py

@@ -0,0 +1,20 @@
+from django.db import models
+from auditable.models import Auditable
+
+
+class UploadedVinsFile(Auditable):
+    filename = models.CharField(max_length=32, unique=True)
+
+    chunk_size = models.IntegerField(default=25000)
+
+    chunks_per_run = models.IntegerField(default=4)
+
+    start_index = models.IntegerField(default=0)
+
+    processed = models.BooleanField(default=False)
+
+    class Meta:
+        db_table = "uploaded_vins_file"
+
+    db_table_comment = "represents a file containing VINs, and parsing information"
+

django/api/services/decoded_vin_record.py

@@ -0,0 +1,73 @@
+from api.models.uploaded_vin_record import UploadedVinRecord
+from api.decoder_constants import get_service
+from api.services.uploaded_vin_record import (
+    set_decode_successful,
+    get_number_of_decode_attempts,
+    set_number_of_decode_attempts,
+)
+from django.db import transaction
+from django.utils import timezone
+
+
+@transaction.atomic
+def save_decoded_data(
+    uploaded_vin_records,
+    vins_to_insert,
+    decoded_records_to_update_map,
+    service_name,
+    decoded_data,
+):
+    decoded_records_to_insert = []
+    decoded_records_to_update = []
+    successful_records = decoded_data["successful_records"]
+    failed_vins = decoded_data["failed_vins"]
+
+    service = get_service(service_name)
+    if service:
+        decoded_vin_model = service.MODEL.value
+        for uploaded_record in uploaded_vin_records:
+            vin = uploaded_record.vin
+            if vin in successful_records:
+                decoded_datum = successful_records.get(vin)
+                set_decode_successful(service_name, uploaded_record, True)
+                if vin in vins_to_insert:
+                    decoded_records_to_insert.append(
+                        decoded_vin_model(vin=vin, data=decoded_datum)
+                    )
+                elif vin in decoded_records_to_update_map:
+                    decoded_record_to_update = decoded_records_to_update_map.get(vin)
+                    decoded_record_to_update.update_timestamp = timezone.now()
+                    decoded_record_to_update.data = decoded_datum
+                    decoded_records_to_update.append(decoded_record_to_update)
+            elif vin in failed_vins:
+                set_decode_successful(service_name, uploaded_record, False)
+
+            set_number_of_decode_attempts(
+                service_name,
+                uploaded_record,
+                get_number_of_decode_attempts(service_name, uploaded_record) + 1,
+            )
+
+        decoded_vin_model.objects.bulk_update(
+            decoded_records_to_update, ["update_timestamp", "data"]
+        )
+        decoded_vin_model.objects.bulk_create(decoded_records_to_insert)
+        UploadedVinRecord.objects.bulk_update(
+            uploaded_vin_records,
+            [
+                "update_timestamp",
+                service.CURRENT_DECODE_SUCCESSFUL.value,
+                service.NUMBER_OF_CURRENT_DECODE_ATTEMPTS.value,
+            ],
+        )
+
+
+def get_decoded_vins(service_name, vins):
+    result = {}
+    service = get_service(service_name)
+    if service:
+        decoded_records_model = service.MODEL.value
+        records = decoded_records_model.objects.filter(vin__in=vins)
+        for record in records:
+            result[record.vin] = record.data
+    return result