Skip to content

Commit

Permalink
Fix file filter bug:
Browse files Browse the repository at this point in the history
There're many 'invalid/unused' members inside a group, so use  all group members count to do cid % operation will filter most files out.
Also it will increase the probability during the probabilityFilter.
So enhance the code to filter out invalid group members
  • Loading branch information
wuhaixian1984 committed Jun 20, 2024
1 parent d336a79 commit 0bd223a
Show file tree
Hide file tree
Showing 2 changed files with 62 additions and 17 deletions.
33 changes: 30 additions & 3 deletions src/tasks/group-info-updater-task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ import { AppContext } from '../types/context';
import { SimpleTask } from '../types/tasks';
import { formatError } from '../utils';
import { makeIntervalTask } from './task-utils';
import { Identity } from '../chain';
import { ValidNodeAnchors } from './node-info-updater-task';
import _ from 'lodash';

async function handleUpdate(context: AppContext, logger: Logger) {
const { api } = context;
Expand All @@ -26,11 +29,35 @@ async function handleUpdate(context: AppContext, logger: Logger) {

// Get group members
const members = await api.groupMembers(groupOwner);
members.sort();
const nodeIndex = members.indexOf(api.getChainAccount());

/// Filter valid members
// First get swork.Identities of the members
const queries = [];
for (const member of members) {
const query = [api.chainApi().query.swork.identities, member];
queries.push(query);
}
const identities = await api.chainApi().queryMulti(queries);

// Perform the filter
const validMembers = [];
for (let i = 0; i < members.length; i++) {
const member = members[i];
const idCodec = identities[i];
if (!_.isNil(idCodec) && !idCodec.isEmpty) {
const id = idCodec.toJSON() as Identity;
const anchor = id.anchor;
if (ValidNodeAnchors.has(anchor)) {
validMembers.push(member);
}
}
}

validMembers.sort();
const nodeIndex = validMembers.indexOf(api.getChainAccount());
context.groupInfo = {
groupAccount: groupOwner,
totalMembers: members.length,
totalMembers: validMembers.length,
nodeIndex,
};
} catch (e) {
Expand Down
46 changes: 32 additions & 14 deletions src/tasks/node-info-updater-task.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,14 @@ import { makeIntervalTask } from './task-utils';
const WorkReportKey =
'0x2e3b7ab5757e6bbf28d3df3b5e01d6b9b7e949778e4650a54fcc65ad1f1ba39f';

export const ValidNodeAnchors = new Set();

async function handleUpdate(context: AppContext, logger: Logger) {
const { api } = context;
try {
let lastKey = null;
let totalCount = 0;
let tempValidNodeAnchors = new Set();
// eslint-disable-next-line
while (true) {
const keys = await (lastKey
Expand All @@ -30,21 +33,33 @@ async function handleUpdate(context: AppContext, logger: Logger) {
.map(extractReportAnchorFromKey)
.filter()
.value();
const workReports = await Bluebird.mapSeries(validKeys, async (k) => {
// logger.info('loading workreport for key: %s', k);
return api.chainApi().query.swork.workReports(k);
});
const validReports = _.filter(workReports, (r) => {
if (!r) {
return false;
}
const report = r.toJSON() as any; // eslint-disable-line
if (!report) {
logger.error('invalid workreport loaded');
return false;

// Get work reports from chain in multi mode
const queries = [];
for (const anchor of validKeys) {
const query = [api.chainApi().query.swork.workReports, anchor];
queries.push(query);
}
const workReports = await api.chainApi().queryMulti(queries);

// Filter out valid reports
const validReports = [];
for (let i = 0; i < validKeys.length; i++) {
const anchor = validKeys[i];
const reportCodec = workReports[i];
if (!_.isNil(reportCodec) && !reportCodec.isEmpty) {
const report = reportCodec.toJSON() as any;
if (!_.isNil(report)) {
if (report.report_slot >= currentSlot - SLOT_LENGTH) {
validReports.push(report);
tempValidNodeAnchors.add(anchor);
}
} else {
logger.error('invalid workreport loaded');
}
}
return report.report_slot >= currentSlot - SLOT_LENGTH;
});
}

logger.info('load %d valid work reports', _.size(validReports));
totalCount += _.size(validReports);
// wait for a short while to reduce system load
Expand All @@ -59,6 +74,9 @@ async function handleUpdate(context: AppContext, logger: Logger) {
context.nodeInfo = {
nodeCount: totalCount,
};
// Update the global ValidNodeAnchors data which will be used by group-info-updater-task
ValidNodeAnchors.clear();
tempValidNodeAnchors.forEach(anchor => ValidNodeAnchors.add(anchor));
} catch (e) {
logger.error('failed updating node info: %s', formatError(e));
}
Expand Down

0 comments on commit 0bd223a

Please sign in to comment.