Retry of failed downloads

Cmdline/Action/Install.cs

@@ -244,11 +244,22 @@ public int RunCommand(CKAN.KSP ksp, object raw_options)
                 user.RaiseMessage(kraken.ToString());
                 return Exit.ERROR;
             }
+            catch (DownloadThrottledKraken kraken)
+            {
+                user.RaiseMessage(kraken.ToString());
+                user.RaiseMessage($"Try the authtoken command. See {kraken.infoUrl} for details.");
+                return Exit.ERROR;
+            }
             catch (DownloadErrorsKraken)
             {
                 user.RaiseMessage("One or more files failed to download, stopped.");
                 return Exit.ERROR;
             }
+            catch (ModuleDownloadErrorsKraken kraken)
+            {
+                user.RaiseMessage(kraken.ToString());
+                return Exit.ERROR;
+            }
             catch (DirectoryNotFoundKraken kraken)
             {
                 user.RaiseMessage("\r\n{0}", kraken.Message);

ConsoleUI/InstallScreen.cs

@@ -94,6 +94,15 @@ public override void Run(Action process = null)
                         RaiseError("Game files reverted.");
                     } catch (DownloadErrorsKraken ex) {
                         RaiseError(ex.ToString());
+                    } catch (ModuleDownloadErrorsKraken ex) {
+                        RaiseError(ex.ToString());
+                    } catch (DownloadThrottledKraken ex) {
+                        if (RaiseYesNoDialog($"{ex.ToString()}\n\nEdit authentication tokens now?")) {
+                            if (ex.infoUrl != null) {
+                                ModInfoScreen.LaunchURL(ex.infoUrl);
+                            }
+                            LaunchSubScreen(new AuthTokenScreen());
+                        }
                     } catch (MissingCertificateKraken ex) {
                         RaiseError(ex.ToString());
                     } catch (InconsistentKraken ex) {

ConsoleUI/ModInfoScreen.cs

@@ -65,7 +65,7 @@ public ModInfoScreen(KSPManager mgr, ChangePlan cp, CkanModule m, bool dbg)
             ));
             AddObject(new ConsoleLabel(
                 13, 5, midL - 2,
-                () => Formatting.FmtSize(mod.download_size)
+                () => CkanModule.FmtSize(mod.download_size)
             ));
             AddObject(new ConsoleLabel(
                 3, 6, midL - 2,

ConsoleUI/ModListScreen.cs

@@ -203,7 +203,7 @@ public ModListScreen(KSPManager mgr, bool dbg)
             // Show total download size of all installed mods
             AddObject(new ConsoleLabel(
                 1, -1, searchWidth,
-                () => $"{Formatting.FmtSize(totalInstalledDownloadSize())} installed",
+                () => $"{CkanModule.FmtSize(totalInstalledDownloadSize())} installed",
                 null,
                 () => ConsoleTheme.Current.DimLabelFg
             ));

ConsoleUI/Toolkit/ConsoleFileMultiSelectDialog.cs

@@ -48,7 +48,7 @@ public ConsoleFileMultiSelectDialog(string title, string startPath, string filPa
 
             AddObject(new ConsoleLabel(
                 left + 2, bottom - 1, right - 2,
-                () => $"{chosenFiles.Count} selected, {Formatting.FmtSize(totalChosenSize())}",
+                () => $"{chosenFiles.Count} selected, {CkanModule.FmtSize(totalChosenSize())}",
                 () => ConsoleTheme.Current.PopupBg,
                 () => ConsoleTheme.Current.PopupFg
             ));
@@ -231,7 +231,7 @@ private string getLength(FileSystemInfo fi)
             } else {
                 FileInfo file = fi as FileInfo;
                 if (file != null) {
-                    return Formatting.FmtSize(file.Length);
+                    return CkanModule.FmtSize(file.Length);
                 } else {
                     return dirSize;
                 }

ConsoleUI/Toolkit/Formatting.cs

@@ -99,27 +99,6 @@ public static List<string> WordWrap(string msg, int w)
             return messageLines;
         }
 
-        /// <summary>
-        /// Format a byte count into readable file size
-        /// </summary>
-        /// <param name="bytes">Number of bytes in a file</param>
-        /// <returns>
-        /// ### bytes or ### KB or ### MB or ### GB
-        /// </returns>
-        public static string FmtSize(long bytes)
-        {
-            const double K = 1024;
-            if (bytes < K) {
-                return $"{bytes} B";
-            } else if (bytes < K * K) {
-                return $"{bytes / K :N1} KB";
-            } else if (bytes < K * K * K) {
-                return $"{bytes / K / K :N1} MB";
-            } else {
-                return $"{bytes / K / K / K :N1} GB";
-            }
-        }
-
     }
 
 }

Core/ModuleInstaller.cs

@@ -168,7 +168,12 @@ public void InstallList(ICollection<CkanModule> modules, RelationshipResolverOpt
             {
                 if (!ksp.Cache.IsCachedZip(module))
                 {
-                    User.RaiseMessage(" * {0} {1} ({2})", module.name, module.version, module.download.Host);
+                    User.RaiseMessage(" * {0} {1} ({2}, {3})",
+                        module.name,
+                        module.version,
+                        module.download.Host,
+                        CkanModule.FmtSize(module.download_size)
+                    );
                     downloads.Add(module);
                 }
                 else

Core/Net/Net.cs

@@ -23,6 +23,14 @@ public class Net
         private static readonly ILog Log = LogManager.GetLogger(typeof (Net));
         private static readonly TxFileManager FileTransaction = new TxFileManager();
 
+        public static readonly Dictionary<string, Uri> ThrottledHosts = new Dictionary<string, Uri>()
+        {
+            {
+                "api.github.com",
+                new Uri("https://github.com/KSP-CKAN/CKAN/wiki/Adding-a-GitHub-API-authtoken")
+            }
+        };
+
         /// <summary>
         ///     Downloads the specified url, and stores it in the filename given.
         ///     If no filename is supplied, a temporary file will be generated.

Core/Net/NetAsyncDownloader.cs

@@ -294,20 +294,37 @@ public void DownloadAndWait(ICollection<Net.DownloadTarget> urls)
             }
 
             // Check to see if we've had any errors. If so, then release the kraken!
-            var exceptions = downloads
-                .Select(x => x.error)
-                .Where(ex => ex != null)
-                .ToList();
-
-            // Let's check if any of these are certificate errors. If so,
-            // we'll report that instead, as this is common (and user-fixable)
-            // under Linux.
-            if (exceptions.Any(ex => ex is WebException &&
-                Regex.IsMatch(ex.Message, "authentication or decryption has failed")))
+            List<KeyValuePair<int, Exception>> exceptions = new List<KeyValuePair<int, Exception>>();
+            for (int i = 0; i < downloads.Count; ++i)
             {
-                throw new MissingCertificateKraken();
+                if (downloads[i].error != null)
+                {
+                    // Check if it's a certificate error. If so, report that instead,
+                    // as this is common (and user-fixable) under Linux.
+                    if (downloads[i].error is WebException)
+                    {
+                        WebException wex = downloads[i].error as WebException;
+                        if (certificatePattern.IsMatch(wex.Message))
+                        {
+                            throw new MissingCertificateKraken();
+                        }
+                        else switch ((wex.Response as HttpWebResponse)?.StatusCode)
+                        {
+                            // Handle HTTP 403 used for throttling
+                            case HttpStatusCode.Forbidden:
+                                Uri infoUrl;
+                                if (Net.ThrottledHosts.TryGetValue(downloads[i].url.Host, out infoUrl))
+                                {
+                                    throw new DownloadThrottledKraken(downloads[i].url, infoUrl);
+                                }
+                                break;
+                        }
+                    }
+                    // Otherwise just note the error and which download it came from,
+                    // then throw them all at once later.
+                    exceptions.Add(new KeyValuePair<int, Exception>(i, downloads[i].error));
+                }
             }
-
             if (exceptions.Count > 0)
             {
                 throw new DownloadErrorsKraken(exceptions);
@@ -316,6 +333,11 @@ public void DownloadAndWait(ICollection<Net.DownloadTarget> urls)
             // Yay! Everything worked!
         }
 
+        private static readonly Regex certificatePattern = new Regex(
+            @"authentication or decryption has failed",
+            RegexOptions.Compiled
+        );
+
         /// <summary>
         /// <see cref="IDownloader.CancelDownload()"/>
         /// This will also call onCompleted with all null arguments.
@@ -387,9 +409,9 @@ private void FileProgressReport(int index, int percent, long bytesDownloaded, lo
             {
                 // Math.Ceiling was added to avoid showing 0 MiB left when finishing
                 User.RaiseProgress(
-                    String.Format("{0} kbps - downloading - {1:f0} MB left",
-                        totalBytesPerSecond/1024,
-                        Math.Ceiling((double)totalBytesLeft/1024/1024)),
+                    String.Format("{0}/sec - downloading - {1} left",
+                        CkanModule.FmtSize(totalBytesPerSecond),
+                        CkanModule.FmtSize(totalBytesLeft)),
                     totalPercentage);
             }
         }
@@ -409,32 +431,36 @@ private void FileDownloadComplete(int index, Exception error)
             {
                 log.InfoFormat("Finished downloading {0}", downloads[index].url);
             }
-            completed_downloads++;
 
             // If there was an error, remember it, but we won't raise it until
             // all downloads are finished or cancelled.
             downloads[index].error = error;
 
-            if (completed_downloads == downloads.Count)
+            if (++completed_downloads == downloads.Count)
             {
-                log.Info("All files finished downloading");
-
-                // If we have a callback, then signal that we're done.
+                FinalizeDownloads();
+            }
+        }
 
-                var fileUrls = new Uri[downloads.Count];
-                var filePaths = new string[downloads.Count];
-                var errors = new Exception[downloads.Count];
+        private void FinalizeDownloads()
+        {
+            log.Info("All files finished downloading");
 
-                for (int i = 0; i < downloads.Count; i++)
-                {
-                    fileUrls[i] = downloads[i].url;
-                    filePaths[i] = downloads[i].path;
-                    errors[i] = downloads[i].error;
-                }
+            Uri[]       fileUrls  = new Uri[downloads.Count];
+            string[]    filePaths = new string[downloads.Count];
+            Exception[] errors    = new Exception[downloads.Count];
 
-                log.Debug("Signalling completion via callback");
-                triggerCompleted(fileUrls, filePaths, errors);
+            for (int i = 0; i < downloads.Count; ++i)
+            {
+                fileUrls[i]  = downloads[i].url;
+                filePaths[i] = downloads[i].path;
+                errors[i]    = downloads[i].error;
             }
+
+            // If we have a callback, then signal that we're done.
+            log.Debug("Signalling completion via callback");
+            triggerCompleted(fileUrls, filePaths, errors);
         }
+
     }
 }

Core/Net/NetAsyncModulesDownloader.cs

@@ -51,20 +51,28 @@ public void DownloadModules(NetModuleCache cache, IEnumerable<CkanModule> module
                 (_uris, paths, errors) =>
                     ModuleDownloadsComplete(cache, _uris, paths, errors);
 
-            // Start the downloads!
-            downloader.DownloadAndWait(
-                unique_downloads.Select(item => new Net.DownloadTarget(
-                    item.Key,
-                    // Use a temp file name
-                    null,
-                    item.Value.download_size,
-                    // Send the MIME type to use for the Accept header
-                    // The GitHub API requires this to include application/octet-stream
-                    string.IsNullOrEmpty(item.Value.download_content_type)
-                        ? defaultMimeType
-                        : $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"
-                )).ToList()
-            );
+            try
+            {
+                // Start the downloads!
+                downloader.DownloadAndWait(
+                    unique_downloads.Select(item => new Net.DownloadTarget(
+                        item.Key,
+                        // Use a temp file name
+                        null,
+                        item.Value.download_size,
+                        // Send the MIME type to use for the Accept header
+                        // The GitHub API requires this to include application/octet-stream
+                        string.IsNullOrEmpty(item.Value.download_content_type)
+                            ? defaultMimeType
+                            : $"{item.Value.download_content_type};q=1.0,{defaultMimeType};q=0.9"
+                    )).ToList()
+                );
+            }
+            catch (DownloadErrorsKraken kraken)
+            {
+                // Associate the errors with the affected modules
+                throw new ModuleDownloadErrorsKraken(this.modules, kraken);
+            }
         }
 
         /// <summary>
@@ -74,33 +82,13 @@ public void DownloadModules(NetModuleCache cache, IEnumerable<CkanModule> module
         /// </summary>
         private void ModuleDownloadsComplete(NetModuleCache cache, Uri[] urls, string[] filenames, Exception[] errors)
         {
-            if (urls != null)
+            if (filenames != null)
             {
-                // spawn up to 3 dialogs
-                int errorDialogsLeft = 3;
-
                 for (int i = 0; i < errors.Length; i++)
                 {
-                    if (errors[i] != null)
-                    {
-                        if (errorDialogsLeft > 0)
-                        {
-                            User.RaiseError("Failed to download \"{0}\" - error: {1}", urls[i], errors[i].Message);
-                            errorDialogsLeft--;
-                        }
-                    }
-                    else
+                    if (errors[i] == null)
                     {
-                        // Even if some of our downloads failed, we want to cache the
-                        // ones which succeeded.
-
-                        // This doesn't work :(
-                        // for some reason the tmp files get deleted before we get here and we get a nasty exception
-                        // not only that but then we try _to install_ the rest of the mods and then CKAN crashes
-                        // and the user's registry gets corrupted forever
-                        // commenting out until this is resolved
-                        // ~ nlight
-
+                        // Cache the downloads that succeeded.
                         try
                         {
                             cache.Store(modules[i], filenames[i], modules[i].StandardName());
@@ -111,14 +99,10 @@ private void ModuleDownloadsComplete(NetModuleCache cache, Uri[] urls, string[]
                         }
                     }
                 }
-            }
 
-            if (filenames != null)
-            {
                 // Finally, remove all our temp files.
                 // We probably *could* have used Store's integrated move function above, but if we managed
                 // to somehow get two URLs the same in our download set, that could cause right troubles!
-
                 foreach (string tmpfile in filenames)
                 {
                     log.DebugFormat("Cleaning up {0}", tmpfile);