Skip to content

show ID for uploads #70

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Hotsapi.Uploader.Common.Test/MockUploader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@ public Task Upload(ReplayFile file)
UploadCallback(file);
return Task.CompletedTask;
}
public async Task<UploadStatus> Upload(string file)
public async Task<IUploadStatus> Upload(string file)
{
await Task.Delay(100);
return UploadStatus.Success;
return UploadStatus.Successful(-1);
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions Hotsapi.Uploader.Common/Analyzer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public Replay Analyze(ReplayFile file)
var status = GetPreStatus(replay, parseResult);

if (status != null) {
file.UploadStatus = status.Value;
file.UploadStatus = status;
}

if (parseResult != DataParser.ReplayParseResult.Success) {
Expand All @@ -44,30 +44,30 @@ public Replay Analyze(ReplayFile file)
}
}

public UploadStatus? GetPreStatus(Replay replay, DataParser.ReplayParseResult parseResult)
public IUploadStatus GetPreStatus(Replay replay, DataParser.ReplayParseResult parseResult)
{
switch (parseResult) {
case DataParser.ReplayParseResult.ComputerPlayerFound:
case DataParser.ReplayParseResult.TryMeMode:
return UploadStatus.AiDetected;
return Rejected.AiDetected;

case DataParser.ReplayParseResult.PTRRegion:
return UploadStatus.PtrRegion;
return Rejected.PtrRegion;

case DataParser.ReplayParseResult.PreAlphaWipe:
return UploadStatus.TooOld;
return Rejected.TooOld;
}

if (parseResult != DataParser.ReplayParseResult.Success) {
return null;
}

if (replay.GameMode == GameMode.Custom) {
return UploadStatus.CustomGame;
return Rejected.CustomGame;
}

if (replay.ReplayBuild < MinimumBuild) {
return UploadStatus.TooOld;
return Rejected.TooOld;
}

return null;
Expand Down
1 change: 1 addition & 0 deletions Hotsapi.Uploader.Common/Hotsapi.Uploader.Common.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
<PropertyGroup>
<TargetFramework>netstandard2.0</TargetFramework>
<Configurations>Debug;Release</Configurations>
<LangVersion>8.0</LangVersion>
</PropertyGroup>

<ItemGroup>
Expand Down
2 changes: 1 addition & 1 deletion Hotsapi.Uploader.Common/IUploader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,6 @@ public interface IUploader
Task CheckDuplicate(IEnumerable<ReplayFile> replays);
Task<int> GetMinimumBuild();
Task Upload(ReplayFile file);
Task<UploadStatus> Upload(string file);
Task<IUploadStatus> Upload(string file);
}
}
64 changes: 49 additions & 15 deletions Hotsapi.Uploader.Common/Manager.cs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@

namespace Hotsapi.Uploader.Common
{
public class StatusCount
{
public StatusCount(string status, int count) => (Status, Count) = (status, count);
public string Status { get; }
public int Count { get; }
}
public class Manager : INotifyPropertyChanged
{
/// <summary>
Expand Down Expand Up @@ -49,16 +55,11 @@ public string Status
}
}

private Dictionary<UploadStatus, int> _aggregates = new Dictionary<UploadStatus, int>();

/// <summary>
/// List of aggregate upload stats
/// </summary>
public Dictionary<UploadStatus, int> Aggregates
{
get {
return _aggregates;
}
}
public IEnumerable<StatusCount> Aggregates { get; private set; } = new List<StatusCount>();

/// <summary>
/// Whether to mark replays for upload to hotslogs
Expand Down Expand Up @@ -103,7 +104,7 @@ public async void Start(IMonitor monitor, IAnalyzer analyzer, IUploader uploader

var replays = ScanReplays();
Files.AddRange(replays);
replays.Where(x => x.UploadStatus == UploadStatus.None).Reverse().Map(x => processingQueue.Add(x));
replays.Where(x => x.UploadStatus == null).Reverse().Map(x => processingQueue.Add(x));

_monitor.ReplayAdded += async (_, e) => {
await EnsureFileAvailable(e.Data, 3000);
Expand Down Expand Up @@ -154,17 +155,50 @@ private async Task UploadLoop()
private void RefreshStatusAndAggregates()
{
_status = Files.Any(x => x.UploadStatus == UploadStatus.InProgress) ? "Uploading..." : "Idle";
_aggregates = Files.GroupBy(x => x.UploadStatus).ToDictionary(x => x.Key, x => x.Count());
Aggregates = Aggregate(Files).Select(pair => new StatusCount(pair.Item1, pair.Item2));
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Status)));
PropertyChanged?.Invoke(this, new PropertyChangedEventArgs(nameof(Aggregates)));
}

//I hate it
private IEnumerable<(String, int)> Aggregate(IEnumerable<ReplayFile> files)
{
var success = 0;
var unhandled = 0;
var rejected = new List<Rejected>();
var failed = 0;
var inProgress = 0;
var errors = 0;
foreach(var file in files) {
var status = file.UploadStatus;
if(status == null) {
unhandled++;
} else {
switch (status) {
case UploadSuccess _: success++; break;
case Rejected r: rejected.Add(r); break;
case InternalError _: errors++; break;
case IFailed _: failed++; break;
case InProgress _: inProgress++; break;
default: errors++; break;
}
}
}
if (unhandled > 0) yield return ("Unhandled", unhandled);
if (inProgress > 0) yield return ("In progress", inProgress);
if (success > 0) yield return ("Success", success);
foreach( var (reason, count) in rejected.GroupBy(r => r.Reason).Select(gr => (gr.Key.ToString(), gr.Count()))) {
yield return ($"Rejected: {reason}", count);
}
if (failed > 0) yield return ("Failed", failed);
}

private void SaveReplayList()
{
try {

// save only replays with fixed status. Will retry failed ones on next launch.
var ignored = new[] { UploadStatus.None, UploadStatus.UploadError, UploadStatus.InProgress };
_storage.Save(Files.Where(x => !ignored.Contains(x.UploadStatus)));
_storage.Save(Files.Where(x => x.UploadStatus != null && (x.UploadStatus.IsSuccess || x.UploadStatus.IsRejected)));
}
catch (Exception ex) {
_log.Error(ex, "Error saving replay list");
Expand Down Expand Up @@ -235,10 +269,10 @@ public async Task EnsureFileAvailable(string filename, int timeout, bool testWri
private bool ShouldDelete(ReplayFile file, Replay replay)
{
return
DeleteAfterUpload.HasFlag(DeleteFiles.PTR) && file.UploadStatus == UploadStatus.PtrRegion ||
DeleteAfterUpload.HasFlag(DeleteFiles.Ai) && file.UploadStatus == UploadStatus.AiDetected ||
DeleteAfterUpload.HasFlag(DeleteFiles.Custom) && file.UploadStatus == UploadStatus.CustomGame ||
file.UploadStatus == UploadStatus.Success && (
DeleteAfterUpload.HasFlag(DeleteFiles.PTR) && file.UploadStatus == Rejected.PtrRegion ||
DeleteAfterUpload.HasFlag(DeleteFiles.Ai) && file.UploadStatus == Rejected.AiDetected ||
DeleteAfterUpload.HasFlag(DeleteFiles.Custom) && file.UploadStatus == Rejected.CustomGame ||
file.UploadStatus.IsSuccess && (
DeleteAfterUpload.HasFlag(DeleteFiles.Brawl) && replay.GameMode == GameMode.Brawl ||
DeleteAfterUpload.HasFlag(DeleteFiles.QuickMatch) && replay.GameMode == GameMode.QuickMatch ||
DeleteAfterUpload.HasFlag(DeleteFiles.UnrankedDraft) && replay.GameMode == GameMode.UnrankedDraft ||
Expand Down
11 changes: 9 additions & 2 deletions Hotsapi.Uploader.Common/ReplayFile.cs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@ public class ReplayFile : INotifyPropertyChanged
public string Fingerprint { get; set; }
public string Filename { get; set; }
public DateTime Created { get; set; }
public int? RemoteID
{
get {
if (UploadStatus is UploadSuccess succ) return succ.UploadID;
else return null;
}
}

private bool _deleted;
public bool Deleted
Expand All @@ -31,8 +38,8 @@ public bool Deleted
}
}

UploadStatus _uploadStatus = UploadStatus.None;
public UploadStatus UploadStatus
IUploadStatus _uploadStatus = null;
public IUploadStatus UploadStatus
{
get {
return _uploadStatus;
Expand Down
82 changes: 75 additions & 7 deletions Hotsapi.Uploader.Common/UploadStatus.cs
Original file line number Diff line number Diff line change
@@ -1,20 +1,88 @@
using System;
using System.Collections.Generic;
using System.Linq;

namespace Hotsapi.Uploader.Common
{
public enum UploadStatus
public interface IUploadStatus
{
bool IsSuccess { get; }
bool IsRejected { get; }
}
public static class UploadStatus
{
public static IUploadStatus InProgress { get; } = new InProgress();
public static IUploadStatus Successful(int id) => new UploadSuccess(id);
}
public readonly struct UploadSuccess : IUploadStatus
{
public bool IsRejected => false;
public bool IsSuccess => true;
public UploadSuccess(int id) => UploadID = id;
public int UploadID { get; }
}
public readonly struct Rejected : IUploadStatus
{
public bool IsRejected => true;
public bool IsSuccess => false;
public RejectionReason Reason { get; }
public string Details { get; }
private Rejected(RejectionReason reason) : this(reason, reason.ToString()) { }
private Rejected(RejectionReason reason, string details) => (Reason, Details) = (reason, details);
public static IUploadStatus AiDetected { get; } = new Rejected(RejectionReason.AiDetected);
public static IUploadStatus PtrRegion { get; } = new Rejected(RejectionReason.PtrRegion);
public static IUploadStatus TooOld { get; } = new Rejected(RejectionReason.TooOld);
public static IUploadStatus CustomGame { get; } = new Rejected(RejectionReason.CustomGame);
public static IUploadStatus Duplicate { get; } = new Rejected(RejectionReason.Duplicate);
public static IUploadStatus ForReason(RejectionReason reason)
{
switch (reason) {
case RejectionReason.AiDetected: return AiDetected;
case RejectionReason.PtrRegion: return PtrRegion;
case RejectionReason.TooOld: return TooOld;
case RejectionReason.CustomGame: return CustomGame;
case RejectionReason.Duplicate: return Duplicate;
default: return new Rejected(reason);
}
}

public static IUploadStatus ForUnknownReason(string reason) => new Rejected(RejectionReason.UnknownReason, reason);
}

/// <summary>
/// Represents a status that shouldn't occur, but allows the application
/// to plod on regardless. Each occurrance of an instance of this class represents
/// a bug in the uploader
/// </summary>
public readonly struct InternalError : IUploadStatus {
public InternalError(string description) => ErrorDescription = description;
public bool IsSuccess => false;
public bool IsRejected => false;
public string ErrorDescription { get; }
}
//for matching
public interface IFailed : IUploadStatus {
public Exception RawException { get; }
}
public readonly struct Failed<E> : IFailed where E : Exception {
public Failed(E cause) => Cause = cause;
public E Cause { get; }
public Exception RawException => Cause;
public bool IsSuccess => false;
public bool IsRejected => false;
}

public readonly struct InProgress : IUploadStatus {
public bool IsSuccess => false;
public bool IsRejected => false;
}

public enum RejectionReason
{
None,
Success,
InProgress,
UploadError,
Duplicate,
AiDetected,
CustomGame,
PtrRegion,
Incomplete,
TooOld,
UnknownReason
}
}
Loading