From b7850da758322446eeddd011650d8535db04f153 Mon Sep 17 00:00:00 2001 From: Ion Dormenco Date: Sun, 15 Oct 2023 09:44:57 +0300 Subject: [PATCH 1/2] Hotfixes * remove limit on answer * save polling station number on polling stations info * add corrupted answers flow for incorrect combos --- .../Commands/FillInAnswerCommand.cs | 4 +- .../Controllers/AnswersController.cs | 4 - .../Handlers/AnswerQueryHandler.cs | 14 +- .../Handlers/FillInAnswerQueryHandler.cs | 141 +- .../Models/CorruptedAnswerDto.cs | 10 + .../Handlers/RegisterPollingSectionHandler.cs | 1 + .../VoteMonitor.Entities/AnswerCorrupted.cs | 15 + .../EfBuilderExtensions.cs | 21 +- src/api/VoteMonitor.Entities/Observer.cs | 1 + .../VoteMonitor.Entities/OptionToQuestion.cs | 1 + .../PollingStationInfoCorrupted.cs | 1 + src/api/VoteMonitor.Entities/VotingContext.cs | 46 +- ...1015061038_AddCorruptedAnswers.Designer.cs | 1331 +++++++++++++++++ .../20231015061038_AddCorruptedAnswers.cs | 115 ++ .../VoteMonitorContextModelSnapshot.cs | 70 +- .../AnswerHandlerTests.cs | 2 +- 16 files changed, 1726 insertions(+), 51 deletions(-) create mode 100644 src/api/VoteMonitor.Api.Answer/Models/CorruptedAnswerDto.cs create mode 100644 src/api/VoteMonitor.Entities/AnswerCorrupted.cs create mode 100644 src/api/VotingIrregularities.Domain.Migrator/Migrations/20231015061038_AddCorruptedAnswers.Designer.cs create mode 100644 src/api/VotingIrregularities.Domain.Migrator/Migrations/20231015061038_AddCorruptedAnswers.cs diff --git a/src/api/VoteMonitor.Api.Answer/Commands/FillInAnswerCommand.cs b/src/api/VoteMonitor.Api.Answer/Commands/FillInAnswerCommand.cs index 34dac3ea..6156bce5 100644 --- a/src/api/VoteMonitor.Api.Answer/Commands/FillInAnswerCommand.cs +++ b/src/api/VoteMonitor.Api.Answer/Commands/FillInAnswerCommand.cs @@ -5,12 +5,14 @@ namespace VoteMonitor.Api.Answer.Commands; public record FillInAnswerCommand : IRequest { - public FillInAnswerCommand(int observerId, IEnumerable answers) + public FillInAnswerCommand(int observerId, IEnumerable answers, IEnumerable corruptedAnswers) { ObserverId = observerId; Answers = answers.ToList().AsReadOnly(); + CorruptedAnswers = corruptedAnswers.ToList().AsReadOnly(); } public int ObserverId { get; } public IReadOnlyCollection Answers { get; } + public IReadOnlyCollection CorruptedAnswers { get; } } diff --git a/src/api/VoteMonitor.Api.Answer/Controllers/AnswersController.cs b/src/api/VoteMonitor.Api.Answer/Controllers/AnswersController.cs index b795da67..f9da7a7a 100644 --- a/src/api/VoteMonitor.Api.Answer/Controllers/AnswersController.cs +++ b/src/api/VoteMonitor.Api.Answer/Controllers/AnswersController.cs @@ -86,10 +86,6 @@ public async Task PostAnswer([FromBody] BulkAnswersRequest answer { // TODO[DH] use a pipeline instead of separate Send commands var command = await _mediator.Send(new Commands.BulkAnswers(this.GetIdObserver(), answerModel.Answers)); - if (!command.Answers.Any()) - { - return NotFound(); - } var result = await _mediator.Send(command); diff --git a/src/api/VoteMonitor.Api.Answer/Handlers/AnswerQueryHandler.cs b/src/api/VoteMonitor.Api.Answer/Handlers/AnswerQueryHandler.cs index 2e60418d..11e73ecb 100644 --- a/src/api/VoteMonitor.Api.Answer/Handlers/AnswerQueryHandler.cs +++ b/src/api/VoteMonitor.Api.Answer/Handlers/AnswerQueryHandler.cs @@ -22,6 +22,7 @@ public AnswerQueryHandler(IPollingStationService pollingPollingStationService, V public async Task Handle(BulkAnswers message, CancellationToken cancellationToken) { var answersBuilder = new List(); + var corruptedAnswersBuilder = new List(); foreach (var answer in message.Answers) { @@ -37,8 +38,19 @@ public async Task Handle(BulkAnswers message, CancellationT CountyCode = answer.CountyCode, }); } + else + { + corruptedAnswersBuilder.Add(new CorruptedAnswerDto() + { + QuestionId = answer.QuestionId, + Options = answer.Options, + PollingStationNumber = answer.PollingStationNumber, + CountyCode = answer.CountyCode, + MunicipalityCode = answer.MunicipalityCode, + }); + } } - var command = new FillInAnswerCommand(message.ObserverId, answersBuilder); + var command = new FillInAnswerCommand(message.ObserverId, answersBuilder, corruptedAnswersBuilder); return command; } } diff --git a/src/api/VoteMonitor.Api.Answer/Handlers/FillInAnswerQueryHandler.cs b/src/api/VoteMonitor.Api.Answer/Handlers/FillInAnswerQueryHandler.cs index f38c89f2..dfb56746 100644 --- a/src/api/VoteMonitor.Api.Answer/Handlers/FillInAnswerQueryHandler.cs +++ b/src/api/VoteMonitor.Api.Answer/Handlers/FillInAnswerQueryHandler.cs @@ -2,6 +2,7 @@ using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; using VoteMonitor.Api.Answer.Commands; +using VoteMonitor.Api.Answer.Models; using VoteMonitor.Entities; namespace VoteMonitor.Api.Answer.Handlers; @@ -22,30 +23,12 @@ public async Task Handle(FillInAnswerCommand message, CancellationToken can try { var lastModified = DateTime.UtcNow; - var newAnswers = GetFlatListOfAnswers(message, lastModified); - var pollingStationIds = message.Answers.Select(a => a.PollingStationId).Distinct().ToList(); using (var tran = await _context.Database.BeginTransactionAsync(cancellationToken)) { - foreach (var pollingStationId in pollingStationIds) - { - var questionIds = message.Answers.Select(a => a.QuestionId).Distinct().ToList(); - - var oldAnswersToBeDeleted = _context.Answers - .Include(a => a.OptionAnswered) - .Where( - a => - a.IdObserver == message.ObserverId && - a.IdPollingStation == pollingStationId) - .WhereRaspunsContains(questionIds) - ; - _context.Answers.RemoveRange(oldAnswersToBeDeleted); - - await _context.SaveChangesAsync(cancellationToken); - } - - await _context.Answers.AddRangeAsync(newAnswers, cancellationToken); + await WriteAnswers(message.ObserverId, message.Answers, lastModified, cancellationToken); + await WriteCorruptedAnswers(message.ObserverId, message.CorruptedAnswers, lastModified, cancellationToken); var result = await _context.SaveChangesAsync(cancellationToken); @@ -56,27 +39,79 @@ public async Task Handle(FillInAnswerCommand message, CancellationToken can } catch (Exception ex) { - _logger.LogError(typeof(FillInAnswerCommand).GetHashCode(), ex, ex.Message); + _logger.LogError(ex, ex.Message); } return await Task.FromResult(-1); } - public static List GetFlatListOfAnswers(FillInAnswerCommand command, DateTime lastModified) + private async Task WriteAnswers(int observerId, IReadOnlyCollection answers, DateTime lastModified, CancellationToken cancellationToken) + { + var newAnswers = GetFlatListOfAnswers(observerId, answers, lastModified); + + var pollingStationIds = answers.Select(a => a.PollingStationId).Distinct().ToList(); + + foreach (var pollingStationId in pollingStationIds) + { + var questionIds = answers.Select(a => a.QuestionId).Distinct().ToList(); + + var oldAnswersToBeDeleted = _context.Answers + .Include(a => a.OptionAnswered) + .Where( + a => + a.IdObserver == observerId && + a.IdPollingStation == pollingStationId) + .WhereRaspunsContains(questionIds) + ; + _context.Answers.RemoveRange(oldAnswersToBeDeleted); + + await _context.SaveChangesAsync(cancellationToken); + } + + await _context.Answers.AddRangeAsync(newAnswers, cancellationToken); + } + private async Task WriteCorruptedAnswers(int observerId, IReadOnlyCollection corruptedAnswers, DateTime lastModified, CancellationToken cancellationToken) + { + var newCorruptedAnswers = GetFlatListOfCorruptedAnswers(observerId, corruptedAnswers, lastModified); + var pollingStationIds = corruptedAnswers.Select(a => (a.CountyCode, a.MunicipalityCode, a.PollingStationNumber)).Distinct().ToList(); + + foreach (var pollingStationId in pollingStationIds) + { + var questionIds = corruptedAnswers.Select(a => a.QuestionId).Distinct().ToList(); + + var oldAnswersToBeDeleted = _context.CorruptedAnswers + .Include(a => a.OptionAnswered) + .Where( + a => + a.IdObserver == observerId && + a.CountyCode == pollingStationId.CountyCode && + a.MunicipalityCode == pollingStationId.MunicipalityCode && + a.PollingStationNumber == pollingStationId.PollingStationNumber) + .WhereRaspunsContains(questionIds); + + _context.CorruptedAnswers.RemoveRange(oldAnswersToBeDeleted); + + await _context.SaveChangesAsync(cancellationToken); + } + + await _context.CorruptedAnswers.AddRangeAsync(newCorruptedAnswers, cancellationToken); + } + + public static List GetFlatListOfAnswers(int observerId, IReadOnlyCollection answers, DateTime lastModified) { - var list = command.Answers.Select(a => new + var list = answers.Select(a => new + { + flat = a.Options.Select(o => new Entities.Answer { - flat = a.Options.Select(o => new Entities.Answer - { - IdObserver = command.ObserverId, - IdPollingStation = a.PollingStationId, - IdOptionToQuestion = o.OptionId, - Value = o.Value, - CountyCode = a.CountyCode, - PollingStationNumber = a.PollingStationNumber, - LastModified = lastModified - }) + IdObserver = observerId, + IdPollingStation = a.PollingStationId, + IdOptionToQuestion = o.OptionId, + Value = o.Value, + CountyCode = a.CountyCode, + PollingStationNumber = a.PollingStationNumber, + LastModified = lastModified }) + }) .SelectMany(a => a.flat) .GroupBy(k => k.IdOptionToQuestion, (g, o) => @@ -85,7 +120,7 @@ public async Task Handle(FillInAnswerCommand message, CancellationToken can return new Entities.Answer { - IdObserver = command.ObserverId, + IdObserver = observerId, IdPollingStation = enumerable.Last().IdPollingStation, IdOptionToQuestion = g, Value = enumerable.Last().Value, @@ -99,4 +134,42 @@ public async Task Handle(FillInAnswerCommand message, CancellationToken can return list; } + + public static List GetFlatListOfCorruptedAnswers(int observerId, IReadOnlyCollection corruptedAnswers, DateTime lastModified) + { + var list = corruptedAnswers.Select(a => new + { + flat = a.Options.Select(o => new Entities.AnswerCorrupted() + { + IdObserver = observerId, + IdOptionToQuestion = o.OptionId, + Value = o.Value, + CountyCode = a.CountyCode, + MunicipalityCode = a.MunicipalityCode, + PollingStationNumber = a.PollingStationNumber, + LastModified = lastModified + }) + }) + .SelectMany(a => a.flat) + .GroupBy(k => k.IdOptionToQuestion, + (g, o) => + { + var enumerable = o as Entities.AnswerCorrupted[] ?? o.ToArray(); + + return new Entities.AnswerCorrupted + { + IdObserver = observerId, + IdOptionToQuestion = g, + Value = enumerable.Last().Value, + CountyCode = enumerable.Last().CountyCode, + MunicipalityCode = enumerable.Last().MunicipalityCode, + PollingStationNumber = enumerable.Last().PollingStationNumber, + LastModified = lastModified + }; + }) + .Distinct() + .ToList(); + + return list; + } } diff --git a/src/api/VoteMonitor.Api.Answer/Models/CorruptedAnswerDto.cs b/src/api/VoteMonitor.Api.Answer/Models/CorruptedAnswerDto.cs new file mode 100644 index 00000000..51a371f9 --- /dev/null +++ b/src/api/VoteMonitor.Api.Answer/Models/CorruptedAnswerDto.cs @@ -0,0 +1,10 @@ +namespace VoteMonitor.Api.Answer.Models; + +public class CorruptedAnswerDto +{ + public int QuestionId { get; set; } + public string CountyCode { get; set; } + public string MunicipalityCode { get; set; } + public int PollingStationNumber { get; set; } + public List Options { get; set; } +} diff --git a/src/api/VoteMonitor.Api.Location/Handlers/RegisterPollingSectionHandler.cs b/src/api/VoteMonitor.Api.Location/Handlers/RegisterPollingSectionHandler.cs index e953eabc..7d121bb0 100644 --- a/src/api/VoteMonitor.Api.Location/Handlers/RegisterPollingSectionHandler.cs +++ b/src/api/VoteMonitor.Api.Location/Handlers/RegisterPollingSectionHandler.cs @@ -100,6 +100,7 @@ private async Task SaveToPollingStationInfoCorruptedData(RegisterPollingSta IdObserver = message.IdObserver, CountyCode = message.CountyCode, MunicipalityCode = message.MunicipalityCode, + PollingStationNumber = message.PollingStationNumber, LastModified = DateTime.UtcNow, ObserverArrivalTime = message.ObserverArrivalTime.AsUtc(), diff --git a/src/api/VoteMonitor.Entities/AnswerCorrupted.cs b/src/api/VoteMonitor.Entities/AnswerCorrupted.cs new file mode 100644 index 00000000..955970a5 --- /dev/null +++ b/src/api/VoteMonitor.Entities/AnswerCorrupted.cs @@ -0,0 +1,15 @@ +namespace VoteMonitor.Entities; + +public class AnswerCorrupted +{ + public int IdObserver { get; set; } + public int IdOptionToQuestion { get; set; } + public DateTime LastModified { get; set; } + public string Value { get; set; } + public string CountyCode { get; set; } + public string MunicipalityCode { get; set; } + public int PollingStationNumber { get; set; } + + public virtual Observer Observer { get; set; } + public virtual OptionToQuestion OptionAnswered { get; set; } +} diff --git a/src/api/VoteMonitor.Entities/EfBuilderExtensions.cs b/src/api/VoteMonitor.Entities/EfBuilderExtensions.cs index 1a77d657..ce03340d 100644 --- a/src/api/VoteMonitor.Entities/EfBuilderExtensions.cs +++ b/src/api/VoteMonitor.Entities/EfBuilderExtensions.cs @@ -1,4 +1,4 @@ -using LinqKit; +using LinqKit; using System.Linq.Expressions; namespace VoteMonitor.Entities; @@ -20,6 +20,23 @@ public static IQueryable WhereRaspunsContains(this IQueryable so ? (a => a.OptionAnswered.IdQuestion == id) : expression.Or(a => a.OptionAnswered.IdQuestion == id)); + return source.Where(ors); + } + /// + /// super simple and dumb translation of .Contains because is not supported pe EF plus + /// this translates to contains in EF SQL + /// + /// + /// + /// + public static IQueryable WhereRaspunsContains(this IQueryable source, IList contains) + { + var ors = contains + .Aggregate>>(null, (expression, id) => + expression == null + ? (a => a.OptionAnswered.IdQuestion == id) + : expression.Or(a => a.OptionAnswered.IdQuestion == id)); + return source.Where(ors); } -} \ No newline at end of file +} diff --git a/src/api/VoteMonitor.Entities/Observer.cs b/src/api/VoteMonitor.Entities/Observer.cs index 3193dbe7..2c058943 100644 --- a/src/api/VoteMonitor.Entities/Observer.cs +++ b/src/api/VoteMonitor.Entities/Observer.cs @@ -16,6 +16,7 @@ public class Observer : IIdentifiableEntity public ICollection Notes { get; set; } = new HashSet(); public ICollection NotesCorrupted { get; set; } = new HashSet(); public ICollection Answers { get; set; } = new HashSet(); + public ICollection CorruptedAnswers { get; set; } = new HashSet(); public ICollection PollingStationInfos { get; set; } = new HashSet(); public ICollection PollingStationInfosCorrupted { get; set; } = new HashSet(); public Ngo Ngo { get; set; } diff --git a/src/api/VoteMonitor.Entities/OptionToQuestion.cs b/src/api/VoteMonitor.Entities/OptionToQuestion.cs index 81670239..d1869910 100644 --- a/src/api/VoteMonitor.Entities/OptionToQuestion.cs +++ b/src/api/VoteMonitor.Entities/OptionToQuestion.cs @@ -13,6 +13,7 @@ public class OptionToQuestion : IIdentifiableEntity public bool Flagged { get; set; } public virtual ICollection Answers { get; } = new HashSet(); + public virtual ICollection CorruptedAnswers { get; } = new HashSet(); public virtual Question Question { get; set; } public virtual Option Option { get; set; } } diff --git a/src/api/VoteMonitor.Entities/PollingStationInfoCorrupted.cs b/src/api/VoteMonitor.Entities/PollingStationInfoCorrupted.cs index 1ffbefd3..026ab951 100644 --- a/src/api/VoteMonitor.Entities/PollingStationInfoCorrupted.cs +++ b/src/api/VoteMonitor.Entities/PollingStationInfoCorrupted.cs @@ -24,4 +24,5 @@ public class PollingStationInfoCorrupted public virtual Observer Observer { get; set; } public string CountyCode { get; set; } public string MunicipalityCode { get; set; } + public int PollingStationNumber { get; set; } } diff --git a/src/api/VoteMonitor.Entities/VotingContext.cs b/src/api/VoteMonitor.Entities/VotingContext.cs index 821873b9..c792c996 100644 --- a/src/api/VoteMonitor.Entities/VotingContext.cs +++ b/src/api/VoteMonitor.Entities/VotingContext.cs @@ -219,8 +219,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.MobileDeviceId).HasMaxLength(500); entity.Property(e => e.Phone) - .IsRequired() - .HasMaxLength(20); + .IsRequired(); entity.Property(e => e.Name) .IsRequired() @@ -288,8 +287,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.LastModified) .HasDefaultValueSql("timezone('utc', now())"); - entity.Property(e => e.Value) - .HasMaxLength(1000); + entity.Property(e => e.Value); entity.Property(e => e.CountyCode) .HasMaxLength(100); @@ -311,6 +309,44 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Restrict); }); + modelBuilder.Entity(entity => + { + entity.HasKey(e => new + { + IdObservator = e.IdObserver, + IdRaspunsDisponibil = e.IdOptionToQuestion, + e.CountyCode, + e.MunicipalityCode, + e.PollingStationNumber, + }); + + entity.HasIndex(e => e.IdObserver); + + entity.HasIndex(e => e.IdOptionToQuestion); + + entity.HasIndex(e => new { e.IdObserver, e.CountyCode, e.MunicipalityCode, e.PollingStationNumber, e.LastModified }); + + entity.Property(e => e.LastModified) + .HasDefaultValueSql("timezone('utc', now())"); + + entity.Property(e => e.Value); + + entity.Property(e => e.MunicipalityCode) + .HasMaxLength(100); + + entity.Property(e => e.CountyCode) + .HasMaxLength(100); + + entity.HasOne(d => d.Observer) + .WithMany(p => p.CorruptedAnswers) + .HasForeignKey(d => d.IdObserver) + .OnDelete(DeleteBehavior.Restrict); + + entity.HasOne(d => d.OptionAnswered) + .WithMany(p => p.CorruptedAnswers) + .HasForeignKey(d => d.IdOptionToQuestion) + .OnDelete(DeleteBehavior.Restrict); + }); modelBuilder.Entity(entity => { @@ -360,6 +396,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) entity.Property(e => e.CountyCode); entity.Property(e => e.MunicipalityCode); + entity.Property(e => e.PollingStationNumber); entity.Property(e => e.NumberOfVotersOnTheList); entity.Property(e => e.NumberOfCommissionMembers); entity.Property(e => e.NumberOfFemaleMembers); @@ -577,6 +614,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) public virtual DbSet Ngos { get; set; } public virtual DbSet