diff --git a/src/gridcoin/project.cpp b/src/gridcoin/project.cpp index cf5c805e18..00e521b555 100644 --- a/src/gridcoin/project.cpp +++ b/src/gridcoin/project.cpp @@ -97,6 +97,8 @@ std::string ProjectEntry::StatusToString(const ProjectEntryStatus& status, const switch(status) { case ProjectEntryStatus::UNKNOWN: return _("Unknown"); case ProjectEntryStatus::DELETED: return _("Deleted"); + case ProjectEntryStatus::MAN_GREYLISTED: return _("Manually Greylisted"); + case ProjectEntryStatus::AUTO_GREYLISTED: return _("Automatically Greylisted"); case ProjectEntryStatus::ACTIVE: return _("Active"); case ProjectEntryStatus::OUT_OF_BOUND: break; } @@ -107,6 +109,8 @@ std::string ProjectEntry::StatusToString(const ProjectEntryStatus& status, const switch(status) { case ProjectEntryStatus::UNKNOWN: return "Unknown"; case ProjectEntryStatus::DELETED: return "Deleted"; + case ProjectEntryStatus::MAN_GREYLISTED: return "Manually Greylisted"; + case ProjectEntryStatus::AUTO_GREYLISTED: return "Automatically Greylisted"; case ProjectEntryStatus::ACTIVE: return "Active"; case ProjectEntryStatus::OUT_OF_BOUND: break; } @@ -196,6 +200,14 @@ Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_ { } +Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, bool manual_greylist) + : ProjectEntry(version, name, url, gdpr_controls, ProjectEntryStatus::UNKNOWN, int64_t {0}) +{ + if (manual_greylist) { + m_status = ProjectEntryStatus::MAN_GREYLISTED; + } +} + Project::Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, int64_t timestamp) : ProjectEntry(version, name, url, gdpr_controls, ProjectEntryStatus::UNKNOWN, timestamp) { @@ -215,8 +227,9 @@ Project::Project(ProjectEntry entry) // Class: WhitelistSnapshot // ----------------------------------------------------------------------------- -WhitelistSnapshot::WhitelistSnapshot(ProjectListPtr projects) +WhitelistSnapshot::WhitelistSnapshot(ProjectListPtr projects, const ProjectEntry::ProjectFilterFlag& filter_used) : m_projects(std::move(projects)) + , m_filter(filter_used) { } @@ -258,6 +271,11 @@ bool WhitelistSnapshot::Contains(const std::string& name) const return false; } +ProjectEntry::ProjectFilterFlag WhitelistSnapshot::FilterUsed() const +{ + return m_filter; +} + WhitelistSnapshot WhitelistSnapshot::Sorted() const { ProjectList sorted(m_projects->begin(), m_projects->end()); @@ -275,26 +293,70 @@ WhitelistSnapshot WhitelistSnapshot::Sorted() const std::sort(sorted.begin(), sorted.end(), ascending_by_name); - return WhitelistSnapshot(std::make_shared(sorted)); + return WhitelistSnapshot(std::make_shared(sorted), m_filter); } // ----------------------------------------------------------------------------- // Class: Whitelist (Registry) // ----------------------------------------------------------------------------- -WhitelistSnapshot Whitelist::Snapshot() const +WhitelistSnapshot Whitelist::Snapshot(const ProjectEntry::ProjectFilterFlag& filter) const { LOCK(cs_lock); ProjectList projects; for (const auto& iter : m_project_entries) { - if (iter.second->m_status == ProjectEntryStatus::ACTIVE) { + switch (filter) { + case ProjectEntry::ProjectFilterFlag::ACTIVE: + if (iter.second->m_status == ProjectEntryStatus::ACTIVE) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::MAN_GREYLISTED: + if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::AUTO_GREYLISTED: + if (iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::GREYLISTED: + if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED + || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::DELETED: + if (iter.second->m_status == ProjectEntryStatus::DELETED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::NOT_ACTIVE: + if (iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED + || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED + || iter.second->m_status == ProjectEntryStatus::DELETED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED: + if (iter.second->m_status == ProjectEntryStatus::ACTIVE + || iter.second->m_status == ProjectEntryStatus::MAN_GREYLISTED + || iter.second->m_status == ProjectEntryStatus::AUTO_GREYLISTED) { + projects.push_back(*iter.second); + } + break; + case ProjectEntry::ProjectFilterFlag::ALL: projects.push_back(*iter.second); + break; + case ProjectEntry::ProjectFilterFlag::NONE: + break; } } - return WhitelistSnapshot(std::make_shared(projects)); + return WhitelistSnapshot(std::make_shared(projects), filter); } void Whitelist::Reset() @@ -325,7 +387,12 @@ void Whitelist::AddDelete(const ContractContext& ctx) // If the contract status is ADD, then ProjectEntryStatus will be ACTIVE. If contract status // is REMOVE then ProjectEntryStatus will be DELETED. if (ctx->m_action == ContractAction::ADD) { + // Normal add/delete contracts are constructed with "unknown" status. The status is recorded on the local client + // based on the add/delete action. The manual greylist status is specified in the contract as constructed, and will carry + // through here for the contract add with the greylist flag set. + if (payload.m_status == ProjectEntryStatus::UNKNOWN) { payload.m_status = ProjectEntryStatus::ACTIVE; + } } else if (ctx->m_action == ContractAction::REMOVE) { payload.m_status = ProjectEntryStatus::DELETED; } @@ -379,8 +446,32 @@ void Whitelist::AddDelete(const ContractContext& ctx) historical.m_hash.GetHex()); } + auto project_iter = m_project_db.find(ctx.m_tx.GetHash()); + // Finally, insert the new project entry (payload) smart pointer into the m_project_entries map. - m_project_entries[payload.m_name] = m_project_db.find(ctx.m_tx.GetHash())->second; + m_project_entries[payload.m_name] = project_iter->second; + + ChangeType status; + + if (project_iter->second->m_status == ProjectEntryStatus::DELETED) { + status = CT_DELETED; + } else if (current_project_entry_present) { + status = CT_UPDATED; + } else { + status = CT_NEW; + } + + NotifyProjectChanged(project_iter->second, status); + + // notify an external script when a project is added to the whitelist, or a project status changes. + #if HAVE_SYSTEM + std::string cmd = gArgs.GetArg("-projectnotify", ""); + + if (!cmd.empty()) { + std::thread t(runCommand, cmd); + t.detach(); // thread runs free + } + #endif return; @@ -545,4 +636,3 @@ template<> const std::string Whitelist::ProjectEntryDB::KeyType() { return std::string("project"); } - diff --git a/src/gridcoin/project.h b/src/gridcoin/project.h index 9e4c55ae87..c63c4d8bd1 100644 --- a/src/gridcoin/project.h +++ b/src/gridcoin/project.h @@ -13,6 +13,7 @@ #include "serialize.h" #include "pubkey.h" #include "sync.h" +#include "node/ui_interface.h" #include #include @@ -31,6 +32,8 @@ namespace GRC //! //! ACTIVE corresponds to an active entry. //! +//! GREYLISTED means that the project temporarily does not meet the whitelist qualification criteria. +//! //! OUT_OF_BOUND must go at the end and be retained for the EnumBytes wrapper. //! enum class ProjectEntryStatus @@ -38,6 +41,8 @@ enum class ProjectEntryStatus UNKNOWN, DELETED, ACTIVE, + MAN_GREYLISTED, + AUTO_GREYLISTED, OUT_OF_BOUND }; @@ -49,6 +54,23 @@ class ProjectEntry //! using Status = EnumByte; + //! + //! \brief Project filter flag enumeration. + //! + //! This controls what project entries by status are in the project whitelist snapshot. + //! + enum ProjectFilterFlag : uint8_t { + NONE = 0b0000, + DELETED = 0b0001, + MAN_GREYLISTED = 0b0010, + AUTO_GREYLISTED = 0b0100, + GREYLISTED = MAN_GREYLISTED | AUTO_GREYLISTED, + ACTIVE = 0b1000, + NOT_ACTIVE = 0b0111, + ALL_BUT_DELETED = 0b1110, + ALL = 0b1111 + }; + //! //! \brief Version number of the current format for a serialized project. //! @@ -284,6 +306,17 @@ class Project : public IContractPayload, public ProjectEntry //! Project(uint32_t version, std::string name, std::string url, bool gdpr_controls); + //! + //! \brief Initialize a \c Project using data from the contract. + //! + //! \param version Project payload version. + //! \param name Project name from contract message key. + //! \param url Project URL from contract message value. + //! \param gdpr_controls Boolean to indicate gdpr stats export controls enforced + //! \param manual_greylist Boolean to force manual greylisting of project + //! + Project(uint32_t version, std::string name, std::string url, bool gdpr_controls, bool manual_greylist); + //! //! \brief Initialize a \c Project using data from the contract. //! @@ -307,7 +340,7 @@ class Project : public IContractPayload, public ProjectEntry Project(std::string name, std::string url, int64_t timestamp, uint32_t version, bool gdpr_controls); //! - //! \brief Initialize a \c Project payloard from a provided project entry + //! \brief Initialize a \c Project payload from a provided project entry //! \param version. Project Payload version. //! \param entry. Project entry to place in the payload. //! @@ -426,8 +459,10 @@ class WhitelistSnapshot //! \brief Initialize a snapshot for the provided project list from the project entry map. //! //! \param projects A copy of the smart pointer to the project list. + //! \param filter_used The project status filter used for the project list. //! - WhitelistSnapshot(ProjectListPtr projects); + WhitelistSnapshot(ProjectListPtr projects, + const ProjectEntry::ProjectFilterFlag& filter_used = ProjectEntry::ProjectFilterFlag::ACTIVE); //! //! \brief Returns an iterator to the beginning. @@ -463,6 +498,13 @@ class WhitelistSnapshot //! bool Contains(const std::string& name) const; + //! + //! \brief Returns the project status filter flag used to populate the snapshot. + //! + //! \return ProjectFilterFlag used for whitelist snapshot. + //! + ProjectEntry::ProjectFilterFlag FilterUsed() const; + //! //! \brief Create a snapshot copy sorted alphabetically by project name. //! @@ -472,6 +514,7 @@ class WhitelistSnapshot private: const ProjectListPtr m_projects; //!< The vector of whitelisted projects. + const ProjectEntry::ProjectFilterFlag m_filter; //!< The filter used to populate the readonly list. }; //! @@ -513,9 +556,10 @@ class Whitelist : public IContractHandler typedef std::map HistoricalProjectEntryMap; //! - //! \brief Get a read-only view of the projects in the whitelist. + //! \brief Get a read-only view of the projects in the whitelist. The default filter is ACTIVE, which + //! provides the original ACTIVE project only view. //! - WhitelistSnapshot Snapshot() const; + WhitelistSnapshot Snapshot(const ProjectEntry::ProjectFilterFlag& filter = ProjectEntry::ProjectFilterFlag::ACTIVE) const; //! //! \brief Destroy the contract handler state to prepare for historical @@ -624,6 +668,11 @@ class Whitelist : public IContractHandler std::set, HistoricalProjectEntryMap> ProjectEntryDB; + //! + //! \brief Core signal to indicate that a project status has changed. + //! + boost::signals2::signal NotifyProjectChanged; + private: //! //! \brief Protects the registry with multithreaded access. This is implemented INTERNAL to the registry class. diff --git a/src/gridcoin/scraper/fwd.h b/src/gridcoin/scraper/fwd.h index d91f9c8eee..19f1a123e2 100644 --- a/src/gridcoin/scraper/fwd.h +++ b/src/gridcoin/scraper/fwd.h @@ -159,6 +159,11 @@ struct ConvergedManifest */ std::vector vExcludedProjects; + //! + //! \brief The list of projects that have been greylisted. + //! + std::vector vGreylistedProjects; + /** Populates the part pointers map in the convergence */ bool PopulateConvergedManifestPartPtrsMap(); diff --git a/src/gridcoin/scraper/scraper.cpp b/src/gridcoin/scraper/scraper.cpp index a58cec9a88..1beac0d001 100755 --- a/src/gridcoin/scraper/scraper.cpp +++ b/src/gridcoin/scraper/scraper.cpp @@ -1571,7 +1571,7 @@ void Scraper(bool bSingleShot) uiInterface.NotifyScraperEvent(scrapereventtypes::Stats, CT_UPDATING, {}); // Get a read-only view of the current project whitelist: - const WhitelistSnapshot projectWhitelist = GetWhitelist().Snapshot(); + const WhitelistSnapshot projectWhitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED); // Delete manifest entries not on whitelist. Take a lock on cs_StructScraperFileManifest for this. { @@ -3636,7 +3636,8 @@ bool ProcessProjectStatsFromStreamByCPID(const std::string& project, boostio::fi // At the individual (byCPIDbyProject) level the AvgRAC is the same as the RAC. statsentry.statsvalue.dAvgRAC = statsentry.statsvalue.dRAC; - // Mag is dealt with on the second pass... so is left at 0.0 on the first pass. + // Mag is dealt with on the second pass, so is set to 0.0 on the first pass. + statsentry.statsvalue.dMag = 0.0; statsentry.statskey.objecttype = statsobjecttype::byCPIDbyProject; statsentry.statskey.objectID = project + "," + cpid; @@ -3654,20 +3655,24 @@ bool ProcessProjectStatsFromStreamByCPID(const std::string& project, boostio::fi // The mScraperStats here is scoped to only this project so we do not need project filtering here. ScraperStats::iterator entry; - for (auto const& entry : mScraperStats) - { - ScraperObjectStats statsentry; + // Statistics tracked for greylisted projects have zero project magnitude, so no need to go through + // and update the CPID level mags. They are all zero if project magnitude is zero. + if (projectmag > 0) { + for (auto const& entry : mScraperStats) + { + ScraperObjectStats statsentry; - statsentry.statskey = entry.first; - statsentry.statsvalue.dTC = entry.second.statsvalue.dTC; - statsentry.statsvalue.dRAT = entry.second.statsvalue.dRAT; - statsentry.statsvalue.dRAC = entry.second.statsvalue.dRAC; - // As per the above the individual (byCPIDbyProject) level the AvgRAC is the same as the RAC. - statsentry.statsvalue.dAvgRAC = entry.second.statsvalue.dAvgRAC; - statsentry.statsvalue.dMag = MagRound(entry.second.statsvalue.dRAC / dProjectRAC * projectmag); + statsentry.statskey = entry.first; + statsentry.statsvalue.dTC = entry.second.statsvalue.dTC; + statsentry.statsvalue.dRAT = entry.second.statsvalue.dRAT; + statsentry.statsvalue.dRAC = entry.second.statsvalue.dRAC; + // As per the above the individual (byCPIDbyProject) level the AvgRAC is the same as the RAC. + statsentry.statsvalue.dAvgRAC = entry.second.statsvalue.dAvgRAC; + statsentry.statsvalue.dMag = MagRound(entry.second.statsvalue.dRAC / dProjectRAC * projectmag); - // Update map entry with the magnitude. - mScraperStats[statsentry.statskey] = statsentry; + // Update map entry with the magnitude. + mScraperStats[statsentry.statskey] = statsentry; + } } // Due to rounding to MAG_ROUND, the actual total project magnitude will not be exactly projectmag, @@ -3802,6 +3807,9 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByCurrentFileManifestState() { _log(logattribute::INFO, "GetScraperStatsByCurrentFileManifestState", "Beginning stats processing."); + // Get a read-only view of the current project greylist + const WhitelistSnapshot greylist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::GREYLISTED); + // Enumerate the count of active projects from the file manifest. Since the manifest is // constructed starting with the whitelist, and then using only the current files, this // will always be less than or equal to the whitelist count from whitelist. @@ -3809,10 +3817,11 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByCurrentFileManifestState() { LOCK(cs_StructScraperFileManifest); - for (auto const& entry : StructScraperFileManifest.mScraperFileManifest) - { - // - if (entry.second.current && !entry.second.excludefromcsmanifest) nActiveProjects++; + for (const auto& entry : StructScraperFileManifest.mScraperFileManifest) { + // Count as active if current, not marked as to be excluded, and also not greylisted. + if (entry.second.current + && !entry.second.excludefromcsmanifest + && !greylist.Contains(entry.second.project)) nActiveProjects++; } } double dMagnitudePerProject = NETWORK_MAGNITUDE / nActiveProjects; @@ -3837,7 +3846,12 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByCurrentFileManifestState() _log(logattribute::INFO, "GetScraperStatsByCurrentFileManifestState", "Processing stats for project: " + project); - LoadProjectFileToStatsByCPID(project, file, dMagnitudePerProject, mProjectScraperStats); + if (!greylist.Contains(entry.second.project)) { + LoadProjectFileToStatsByCPID(project, file, dMagnitudePerProject, mProjectScraperStats); + } else { + // Project magnitude for a greylisted project is zero. + LoadProjectFileToStatsByCPID(project, file, 0.0, mProjectScraperStats); + } // Insert into overall map. for (auto const& entry2 : mProjectScraperStats) @@ -3874,6 +3888,9 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge { _log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Beginning stats processing."); + // Get a read-only view of the current project greylist + const WhitelistSnapshot greylist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::GREYLISTED); + ScraperStatsAndVerifiedBeacons stats_and_verified_beacons; // Enumerate the count of active projects from the dummy converged manifest. One of the parts @@ -3903,6 +3920,14 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge stats_and_verified_beacons.mVerifiedMap = VerifiedBeaconMap; unsigned int nActiveProjects = StructConvergedManifest.ConvergedManifestPartPtrsMap.size() - exclude_parts_from_count; + + // If a project part is greylisted, do not count it as an active project, even though stats have been collected. + for (const auto& project : StructConvergedManifest.ConvergedManifestPartPtrsMap) { + if (greylist.Contains(project.first)) { + --nActiveProjects; + } + } + _log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Number of active projects in converged manifest = " + ToString(nActiveProjects)); @@ -3922,7 +3947,12 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsByConvergedManifest(const Converge { _log(logattribute::INFO, "GetScraperStatsByConvergedManifest", "Processing stats for project: " + project); - LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); + if (!greylist.Contains(project)) { + LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); + } else { + // Project magnitude for a greylisted project is zero. + LoadProjectObjectToStatsByCPID(project, entry->second->data, 0.0, mProjectScraperStats); + } // Insert into overall map. for (auto const& entry2 : mProjectScraperStats) @@ -3945,6 +3975,9 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes { _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Beginning stats processing."); + // Get a read-only view of the current project greylist + const WhitelistSnapshot greylist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::GREYLISTED); + // Create a dummy converged manifest and fill out the dummy ConvergedManifest structure from the provided // manifest. ConvergedManifest StructDummyConvergedManifest(manifest); @@ -3979,6 +4012,14 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes unsigned int nActiveProjects = StructDummyConvergedManifest.ConvergedManifestPartPtrsMap.size() - exclude_parts_from_count; + + // If a project part is greylisted, do not count it as an active project, even though stats have been collected. + for (const auto& project : StructDummyConvergedManifest.ConvergedManifestPartPtrsMap) { + if (greylist.Contains(project.first)) { + --nActiveProjects; + } + } + _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Number of active projects in converged manifest = " + ToString(nActiveProjects)); @@ -3995,7 +4036,12 @@ ScraperStatsAndVerifiedBeacons GetScraperStatsFromSingleManifest(CScraperManifes { _log(logattribute::INFO, "GetScraperStatsFromSingleManifest", "Processing stats for project: " + project); - LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); + if (!greylist.Contains(project)) { + LoadProjectObjectToStatsByCPID(project, entry->second->data, dMagnitudePerProject, mProjectScraperStats); + } else { + // Project magnitude for a greylisted project is zero. + LoadProjectObjectToStatsByCPID(project, entry->second->data, 0.0, mProjectScraperStats); + } // Insert into overall map. stats_and_verified_beacons.mScraperStats.insert(mProjectScraperStats.begin(), mProjectScraperStats.end()); @@ -4855,7 +4901,7 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes // Get a read-only view of the current project whitelist to fill out the // excluded projects vector later on: - const WhitelistSnapshot projectWhitelist = GetWhitelist().Snapshot(); + const WhitelistSnapshot projectWhitelist = GetWhitelist().Snapshot(GRC::ProjectEntry::ProjectFilterFlag::ALL_BUT_DELETED); if (bConvergenceSuccessful) { @@ -4884,6 +4930,12 @@ bool ScraperConstructConvergedManifest(ConvergedManifest& StructConvergedManifes // to try and recover project by project. for (const auto& iProjects : projectWhitelist) { + // If project is greylisted, push project name to greylist vector. + if (iProjects.m_status == GRC::ProjectEntryStatus::MAN_GREYLISTED + || iProjects.m_status == GRC::ProjectEntryStatus::AUTO_GREYLISTED) { + StructConvergedManifest.vGreylistedProjects.push_back(iProjects.m_name); + } + if (StructConvergedManifest.ConvergedManifestPartPtrsMap.find(iProjects.m_name) == StructConvergedManifest.ConvergedManifestPartPtrsMap.end()) { @@ -4967,6 +5019,12 @@ bool ScraperConstructConvergedManifestByProject(const WhitelistSnapshot& project for (const auto& iWhitelistProject : projectWhitelist) { + // If project is greylisted, push project name to greylist vector for later use. + if (iWhitelistProject.m_status == GRC::ProjectEntryStatus::MAN_GREYLISTED + ||iWhitelistProject.m_status == GRC::ProjectEntryStatus::AUTO_GREYLISTED) { + StructConvergedManifest.vGreylistedProjects.push_back(iWhitelistProject.m_name); + } + // Do a map for unique ProjectObject times ordered by descending time then content hash. Note that for Project // Objects (Parts), the content hash is the object hash. We also need the consensus block here, because we are // "composing" the manifest by parts, so we will need to choose the latest consensus block by manifest time. This @@ -5138,8 +5196,10 @@ bool ScraperConstructConvergedManifestByProject(const WhitelistSnapshot& project auto convergence_by_project_ratio = [](){ LOCK(cs_ScraperGlobals); return CONVERGENCE_BY_PROJECT_RATIO; }; - // If we meet the rule of CONVERGENCE_BY_PROJECT_RATIO, then proceed to fill out the rest of the map. - if ((double)iCountSuccessfulConvergedProjects / (double)projectWhitelist.size() >= convergence_by_project_ratio()) + // If we meet the rule of CONVERGENCE_BY_PROJECT_RATIO, then proceed to fill out the rest of the map. Note that the greylisted + // projects are excluded from the count in the denominator as it is not expected to necessarily achieve convergence on those. + if ((double)iCountSuccessfulConvergedProjects / + (double)(projectWhitelist.size() - StructConvergedManifest.vGreylistedProjects.size()) >= convergence_by_project_ratio()) { AppCacheSection mScrapers = GetScrapersCache(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 69d73a2a39..e88022d219 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2167,23 +2167,35 @@ UniValue superblocks(const UniValue& params, bool fHelp) //! //! To whitelist a project: //! -//! addkey add project projectname url +//! addkey add project projectname url (< v2) //! -//! To de-whitelist a project: +//! addkey add project projectname url gdpr_controls (v2) +//! +//! addkey add project projectname url gdpr_controls manual_greylist (v3) //! -//! addkey delete project projectname +//! To de-whitelist a project: //! -//! or +//! addkey delete project projectname 1 (<= v2) //! -//! addkey delete project projectname 1 +//! addkey delete project projectname (>= v3) //! //! Key examples: //! +//! To add: +//! //! v1: addkey add project milkyway@home http://milkyway.cs.rpi.edu/milkyway/@ //! v2: addkey add project milkyway@home http://milkyway.cs.rpi.edu/milkyway/@ true +//! +//! To manually greylist: +//! +//! v3: addkey add project milkyway@home http://milkyway.cs.rpi.edu/milkyway/@ true true +//! +//! To delete: +//! //! v1 or v2: addkey delete project milkyway@home //! v1 or v2: addkey delete project milkyway@home 1 (last parameter is a dummy) //! v2: addkey delete project milkyway@home 1 false (last two parameters are dummies) +//! v3: addkey delete project milkyway@home (the dummy parameters are no longer needed for v3 deletes) //! //! GRC will only memorize the *last* value it finds for a key in the highest //! block. @@ -2227,7 +2239,8 @@ UniValue addkey(const UniValue& params, bool fHelp) && action == GRC::ContractAction::ADD && type == GRC::ContractType::PROJECT) { required_param_count = 5; - param_count_max = 5; + + block_v13_enabled ? param_count_max = 6 : param_count_max = 5; } if ((type == GRC::ContractType::PROJECT || type == GRC::ContractType::SCRAPER) @@ -2305,6 +2318,7 @@ UniValue addkey(const UniValue& params, bool fHelp) { if (action == GRC::ContractAction::ADD) { bool gdpr_export_control = false; + bool manually_greylist = false; if (block_v13_enabled) { // We must do our own conversion to boolean here, because the 5th parameter can either be @@ -2317,13 +2331,20 @@ UniValue addkey(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_PARAMETER, "GDPR export parameter invalid. Must be true or false."); } + if (params.size() == 6) { + if (ToLower(params[4].get_str()) == "true") { + manually_greylist = true; + } + } + contract = GRC::MakeContract( contract_version, action, uint32_t{3}, // Contract payload version number, 3 params[2].get_str(), // Name params[3].get_str(), // URL - gdpr_export_control); // GDPR stats export protection enforced boolean + gdpr_export_control, // GDPR stats export protection enforced boolean + manually_greylist); // manual greylist flag } else if (project_v2_enabled) { // We must do our own conversion to boolean here, because the 5th parameter can either be @@ -2359,7 +2380,8 @@ UniValue addkey(const UniValue& params, bool fHelp) uint32_t{3}, // Contract payload version number, 3 params[2].get_str(), // Name std::string{}, // URL ignored - false); // GDPR status irrelevant + false, // GDPR status irrelevant + false); // manual greylisting irrelevant } else if (project_v2_enabled) { contract = GRC::MakeContract( @@ -2570,15 +2592,23 @@ UniValue debug(const UniValue& params, bool fHelp) UniValue listprojects(const UniValue& params, bool fHelp) { - if (fHelp || params.size() != 0) + if (fHelp || params.size() > 1) throw runtime_error( - "listprojects\n" + "listprojects \n" + "\n" + " -> true to show all projects, including greylisted and deleted. Defaults to false.\n" "\n" "Displays information about whitelisted projects.\n"); UniValue res(UniValue::VOBJ); - for (const auto& project : GRC::GetWhitelist().Snapshot().Sorted()) { + GRC::Project::ProjectFilterFlag filter = GRC::Project::ProjectFilterFlag::ACTIVE; + + if (params.size() && params[0].get_bool() == true) { + filter = GRC::Project::ProjectFilterFlag::ALL; + } + + for (const auto& project : GRC::GetWhitelist().Snapshot(filter).Sorted()) { UniValue entry(UniValue::VOBJ); entry.pushKV("version", (int)project.m_version); @@ -2593,6 +2623,7 @@ UniValue listprojects(const UniValue& params, bool fHelp) } entry.pushKV("time", DateTimeStrFormat(project.m_timestamp)); + entry.pushKV("status", project.StatusToString()); res.pushKV(project.m_name, entry); } diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f99bc02991..689aa6134c 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -214,6 +214,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getblockstats" , 2 }, { "inspectaccrualsnapshot" , 0 }, { "listmanifests" , 0 }, + { "listprojects" , 0 }, { "sendalert" , 2 }, { "sendalert" , 3 }, { "sendalert" , 4 },