scraper, project: Implement automated greylisting (tracking PR for WIP)

src/gridcoin/project.cpp

@@ -97,6 +97,8 @@ std::string ProjectEntry::StatusToString(const ProjectEntryStatus& status, const
         switch(status) {
         case ProjectEntryStatus::UNKNOWN:         return _("Unknown");
         case ProjectEntryStatus::DELETED:         return _("Deleted");
+        case ProjectEntryStatus::MAN_GREYLISTED:  return _("Manually Greylisted");
+        case ProjectEntryStatus::AUTO_GREYLISTED: return _("Automatically Greylisted");
         case ProjectEntryStatus::ACTIVE:          return _("Active");
         case ProjectEntryStatus::OUT_OF_BOUND:    break;
         }
@@ -107,6 +109,8 @@ std::string ProjectEntry::StatusToString(const ProjectEntryStatus& status, const
         switch(status) {
         case ProjectEntryStatus::UNKNOWN:         return "Unknown";
         case ProjectEntryStatus::DELETED:         return "Deleted";
+        case ProjectEntryStatus::MAN_GREYLISTED:  return "Manually Greylisted";
+        case ProjectEntryStatus::AUTO_GREYLISTED: return "Automatically Greylisted";
         case ProjectEntryStatus::ACTIVE:          return "Active";
         case ProjectEntryStatus::OUT_OF_BOUND:    break;
         }
@@ -196,6 +200,14 @@ Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_
 {
 }
 
+Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, bool manual_greylist)
+    : ProjectEntry(version, name, url, gdpr_controls, ProjectEntryStatus::UNKNOWN, int64_t {0})
+{
+    if (manual_greylist) {
+        m_status = ProjectEntryStatus::MAN_GREYLISTED;
+    }
+}
+
 Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, int64_t timestamp)
     : ProjectEntry(version, name, url, gdpr_controls, ProjectEntryStatus::UNKNOWN, timestamp)
 {
@@ -215,8 +227,9 @@ Project::Project(ProjectEntry entry)
 // Class: WhitelistSnapshot
 // -----------------------------------------------------------------------------
 
-WhitelistSnapshot::WhitelistSnapshot(ProjectListPtr projects)
+WhitelistSnapshot::WhitelistSnapshot(ProjectListPtr projects, const ProjectEntry::ProjectFilterFlag& filter_used)
     : m_projects(std::move(projects))
+      , m_filter(filter_used)
 {
 }
 
@@ -258,6 +271,11 @@ bool WhitelistSnapshot::Contains(const std::string& name) const
     return false;
 }
 
+ProjectEntry::ProjectFilterFlag WhitelistSnapshot::FilterUsed() const
+{
+    return m_filter;
+}
+
 WhitelistSnapshot WhitelistSnapshot::Sorted() const
 {
     ProjectList sorted(m_projects->begin(), m_projects->end());
@@ -275,26 +293,70 @@ WhitelistSnapshot WhitelistSnapshot::Sorted() const
 
     std::sort(sorted.begin(), sorted.end(), ascending_by_name);
 
-    return WhitelistSnapshot(std::make_shared<ProjectList>(sorted));
+    return WhitelistSnapshot(std::make_shared<ProjectList>(sorted), m_filter);
 }
 
 // -----------------------------------------------------------------------------
 // Class: Whitelist (Registry)
 // -----------------------------------------------------------------------------
 
-WhitelistSnapshot Whitelist::Snapshot() const
+WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter) const
 {
     LOCK(cs_lock);
 
     ProjectList projects;
 
     for (const auto& iter : m_project_entries) {
-        if (iter.second->m_status == ProjectEntryStatus::ACTIVE) {
+        switch (filter) {
+        case ProjectEntry::ProjectFilterFlag::ACTIVE:
+            if (iter.second->m_status == ProjectEntryStatus::ACTIVE) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::MAN_GREYLISTED:
+            if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::AUTO_GREYLISTED:
+            if (iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::GREYLISTED:
+            if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED
+                || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::DELETED:
+            if (iter.second->m_status == ProjectEntryStatus::DELETED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::NOT_ACTIVE:
+            if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED
+                || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED
+                || iter.second->m_status == ProjectEntryStatus::DELETED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED:
+            if (iter.second->m_status == ProjectEntryStatus::ACTIVE
+                || iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED
+                || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) {
+                projects.push_back(*iter.second);
+            }
+            break;
+        case ProjectEntry::ProjectFilterFlag::ALL:
             projects.push_back(*iter.second);
+            break;
+        case ProjectEntry::ProjectFilterFlag::NONE:
+            break;
         }
     }
 
-    return WhitelistSnapshot(std::make_shared<ProjectList>(projects));
+    return WhitelistSnapshot(std::make_shared<ProjectList>(projects), filter);
 }
 
 void Whitelist::Reset()
@@ -325,7 +387,12 @@ void Whitelist::AddDelete(const ContractContext& ctx)
     // If the contract status is ADD, then ProjectEntryStatus will be ACTIVE. If contract status
     // is REMOVE then ProjectEntryStatus will be DELETED.
     if (ctx->m_action == ContractAction::ADD) {
+        // Normal add/delete contracts are constructed with "unknown" status. The status is recorded on the local client
+        // based on the add/delete action. The manual greylist status is specified in the contract as constructed, and will carry
+        // through here for the contract add with the greylist flag set.
+        if (payload.m_status == ProjectEntryStatus::UNKNOWN) {
         payload.m_status = ProjectEntryStatus::ACTIVE;
+        }
     } else if (ctx->m_action == ContractAction::REMOVE) {
         payload.m_status = ProjectEntryStatus::DELETED;
     }
@@ -379,8 +446,32 @@ void Whitelist::AddDelete(const ContractContext& ctx)
                  historical.m_hash.GetHex());
     }
 
+    auto project_iter = m_project_db.find(ctx.m_tx.GetHash());
+
     // Finally, insert the new project entry (payload) smart pointer into the m_project_entries map.
-    m_project_entries[payload.m_name] = m_project_db.find(ctx.m_tx.GetHash())->second;
+    m_project_entries[payload.m_name] = project_iter->second;
+
+    ChangeType status;
+
+    if (project_iter->second->m_status == ProjectEntryStatus::DELETED) {
+        status = CT_DELETED;
+    } else if (current_project_entry_present) {
+        status = CT_UPDATED;
+    } else {
+        status = CT_NEW;
+    }
+
+    NotifyProjectChanged(project_iter->second, status);
+
+    // notify an external script when a project is added to the whitelist, or a project status changes.
+    #if HAVE_SYSTEM
+    std::string cmd = gArgs.GetArg("-projectnotify", "");
+
+    if (!cmd.empty()) {
+        std::thread t(runCommand, cmd);
+        t.detach(); // thread runs free
+    }
+    #endif
 
     return;
 
@@ -545,4 +636,3 @@ template<> const std::string Whitelist::ProjectEntryDB::KeyType()
 {
     return std::string("project");
 }
-

src/gridcoin/project.h

@@ -13,6 +13,7 @@
 #include "serialize.h"
 #include "pubkey.h"
 #include "sync.h"
+#include "node/ui_interface.h"
 
 #include <memory>
 #include <vector>
@@ -31,13 +32,17 @@ namespace GRC
 //!
 //! ACTIVE corresponds to an active entry.
 //!
+//! GREYLISTED means that the project temporarily does not meet the whitelist qualification criteria.
+//!
 //! OUT_OF_BOUND must go at the end and be retained for the EnumBytes wrapper.
 //!
 enum class ProjectEntryStatus
 {
     UNKNOWN,
     DELETED,
     ACTIVE,
+    MAN_GREYLISTED,
+    AUTO_GREYLISTED,
     OUT_OF_BOUND
 };
 
@@ -49,6 +54,23 @@ class ProjectEntry
     //!
     using Status = EnumByte<ProjectEntryStatus>;
 
+    //!
+    //! \brief Project filter flag enumeration.
+    //!
+    //! This controls what project entries by status are in the project whitelist snapshot.
+    //!
+    enum ProjectFilterFlag : uint8_t {
+        NONE            = 0b0000,
+        DELETED         = 0b0001,
+        MAN_GREYLISTED  = 0b0010,
+        AUTO_GREYLISTED = 0b0100,
+        GREYLISTED      = MAN_GREYLISTED | AUTO_GREYLISTED,
+        ACTIVE          = 0b1000,
+        NOT_ACTIVE      = 0b0111,
+        ALL_BUT_DELETED = 0b1110,
+        ALL             = 0b1111
+    };
+
     //!
     //! \brief Version number of the current format for a serialized project.
     //!
@@ -284,6 +306,17 @@ class Project : public IContractPayload, public ProjectEntry
     //!
     Project(uint32_t version, std::string name, std::string url, bool gdpr_controls);
 
+    //!
+    //! \brief Initialize a \c Project using data from the contract.
+    //!
+    //! \param version         Project payload version.
+    //! \param name            Project name from contract message key.
+    //! \param url             Project URL from contract message value.
+    //! \param gdpr_controls   Boolean to indicate gdpr stats export controls enforced
+    //! \param manual_greylist Boolean to force manual greylisting of project
+    //!
+    Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, bool manual_greylist);
+
     //!
     //! \brief Initialize a \c Project using data from the contract.
     //!
@@ -307,7 +340,7 @@ class Project : public IContractPayload, public ProjectEntry
     Project(std::string name, std::string url, int64_t timestamp, uint32_t version, bool gdpr_controls);
 
     //!
-    //! \brief Initialize a \c Project payloard from a provided project entry
+    //! \brief Initialize a \c Project payload from a provided project entry
     //! \param version. Project Payload version.
     //! \param entry. Project entry to place in the payload.
     //!
@@ -426,8 +459,10 @@ class WhitelistSnapshot
     //! \brief Initialize a snapshot for the provided project list from the project entry map.
     //!
     //! \param projects A copy of the smart pointer to the project list.
+    //! \param filter_used The project status filter used for the project list.
     //!
-    WhitelistSnapshot(ProjectListPtr projects);
+    WhitelistSnapshot(ProjectListPtr projects,
+                      const ProjectEntry::ProjectFilterFlag& filter_used = ProjectEntry::ProjectFilterFlag::ACTIVE);
 
     //!
     //! \brief Returns an iterator to the beginning.
@@ -463,6 +498,13 @@ class WhitelistSnapshot
     //!
     bool Contains(const std::string& name) const;
 
+    //!
+    //! \brief Returns the project status filter flag used to populate the snapshot.
+    //!
+    //! \return ProjectFilterFlag used for whitelist snapshot.
+    //!
+    ProjectEntry::ProjectFilterFlag FilterUsed() const;
+
     //!
     //! \brief Create a snapshot copy sorted alphabetically by project name.
     //!
@@ -472,6 +514,7 @@ class WhitelistSnapshot
 
 private:
     const ProjectListPtr m_projects;  //!< The vector of whitelisted projects.
+    const ProjectEntry::ProjectFilterFlag m_filter; //!< The filter used to populate the readonly list.
 };
 
 //!
@@ -513,9 +556,10 @@ class Whitelist : public IContractHandler
     typedef std::map<uint256, ProjectEntry_ptr> HistoricalProjectEntryMap;
 
     //!
-    //! \brief Get a read-only view of the projects in the whitelist.
+    //! \brief Get a read-only view of the projects in the whitelist. The default filter is ACTIVE, which
+    //! provides the original ACTIVE project only view.
     //!
-    WhitelistSnapshot Snapshot() const;
+    WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE) const;
 
     //!
     //! \brief Destroy the contract handler state to prepare for historical
@@ -624,6 +668,11 @@ class Whitelist : public IContractHandler
                        std::set<ProjectEntry>,
                        HistoricalProjectEntryMap> ProjectEntryDB;
 
+    //!
+    //! \brief Core signal to indicate that a project status has changed.
+    //!
+    boost::signals2::signal<void (const ProjectEntry_ptr project, ChangeType status)> NotifyProjectChanged;
+
 private:
     //!
     //! \brief Protects the registry with multithreaded access. This is implemented INTERNAL to the registry class.

src/gridcoin/scraper/fwd.h

@@ -159,6 +159,11 @@ struct ConvergedManifest
      */
     std::vector<std::string> vExcludedProjects;
 
+    //!
+    //! \brief The list of projects that have been greylisted.
+    //!
+    std::vector<std::string> vGreylistedProjects;
+
     /** Populates the part pointers map in the convergence */
     bool PopulateConvergedManifestPartPtrsMap();