Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@benzekrimaha
benzekrimaha committed Jan 23, 2025
1 parent 9e08f7a commit 9325b3f
Show file tree
Hide file tree
Showing 19 changed files with 2,697 additions and 2,591 deletions.
8 changes: 7 additions & 1 deletion .eslintrc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,13 @@
"no-lonely-if": 0,
"max-classes-per-file": 0,
"prefer-spread": 0,
"no-constructor-return": 0
"no-constructor-return": 0,
"no-console": 0,
"object-curly-spacing": 0,
"no-extend-native": 0
},
"globals": {
"BigInt": "readonly"
}
}

4 changes: 2 additions & 2 deletions .github/workflows/test.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ env:

jobs:
prepare:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
permissions:
# Need to explicitely add package write permissions for dependabot
contents: read
Expand Down Expand Up @@ -45,7 +45,7 @@ jobs:

tests:
needs: prepare
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
services:
mongodb:
image: ghcr.io/${{ github.repository }}/ci-mongodb:${{ github.sha }}
Expand Down
18 changes: 13 additions & 5 deletions CountItems/CountManager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,12 +11,12 @@ class CountManager {
this.maxConcurrent = params.maxConcurrent;
this.temporaryStore = {};
this.store = {
objects: 0,
versions: 0,
objects: BigInt(0),
versions: BigInt(0),
buckets: 0,
bucketList: [],
dataManaged: {
total: { curr: 0, prev: 0 },
total: { curr: BigInt(0), prev: BigInt(0) },
byLocation: {},
},
stalled: 0,
Expand Down Expand Up @@ -61,14 +61,22 @@ class CountManager {
if (!results) {
return;
}
this.store.versions += results.versions;
this.store.objects += results.objects;
this.store.versions += results.versions ? results.versions : BigInt(0);
// eslint-disable-next-line no-console
console.log('results.objects', results.objects);
this.store.objects += results.objects ? results.objects : BigInt(0);
// eslint-disable-next-line no-console
console.log('this.store.objects', this.store.objects);
this.store.stalled += results.stalled;
if (results.dataManaged
&& results.dataManaged.locations
&& results.dataManaged.total) {
const { locations, total } = results.dataManaged;
// eslint-disable-next-line no-console
console.log('total.curr', total.curr);
this.store.dataManaged.total.curr += total.curr;
// eslint-disable-next-line no-console
console.log('total.prev', total.prev);
this.store.dataManaged.total.prev += total.prev;
Object.keys(locations).forEach(site => {
if (!this.store.dataManaged.byLocation[site]) {
Expand Down
4 changes: 4 additions & 0 deletions CountItems/CountWorker.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ const async = require('async');
const { BucketInfo } = require('arsenal').models;
const monitoring = require('../utils/monitoring');

/* eslint-disable no-extend-native */
BigInt.prototype.toJSON = () => this.toString();
/* eslint-disable no-extend-native */

class CountWorker {
constructor(params) {
this.log = params.log;
Expand Down
4 changes: 4 additions & 0 deletions CountItems/masterProcess.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,10 @@ const loggerConfig = {

let waitingForPromScraping = false;

/* eslint-disable no-extend-native */
BigInt.prototype.toJSON = () => this.toString();
/* eslint-disable no-extend-native */

werelogs.configure(loggerConfig);
const log = new werelogs.Logger('S3Utils::CountItems::Master');

Expand Down
88 changes: 44 additions & 44 deletions CountItems/utils/utils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,75 +9,75 @@ function consolidateDataMetrics(target, source) {
if (!resTarget.usedCapacity) {
Object.assign(resTarget, {
usedCapacity: {
current: 0,
nonCurrent: 0,
_currentCold: 0,
_nonCurrentCold: 0,
_currentRestored: 0,
_currentRestoring: 0,
_nonCurrentRestored: 0,
_nonCurrentRestoring: 0,
_inflightsPreScan: 0,
_incompleteMPUParts: 0,
current: BigInt(0),
nonCurrent: BigInt(0),
_currentCold: BigInt(0),
_nonCurrentCold: BigInt(0),
_currentRestored: BigInt(0),
_currentRestoring: BigInt(0),
_nonCurrentRestored: BigInt(0),
_nonCurrentRestoring: BigInt(0),
_inflightsPreScan: BigInt(0),
_incompleteMPUParts: BigInt(0),
},
});
}
if (!resTarget.objectCount) {
Object.assign(resTarget, {
objectCount: {
current: 0,
nonCurrent: 0,
_currentCold: 0,
_nonCurrentCold: 0,
_currentRestored: 0,
_currentRestoring: 0,
_nonCurrentRestored: 0,
_nonCurrentRestoring: 0,
_incompleteMPUUploads: 0,
deleteMarker: 0,
current: BigInt(0),
nonCurrent: BigInt(0),
_currentCold: BigInt(0),
_nonCurrentCold: BigInt(0),
_currentRestored: BigInt(0),
_currentRestoring: BigInt(0),
_nonCurrentRestored: BigInt(0),
_nonCurrentRestoring: BigInt(0),
_incompleteMPUUploads: BigInt(0),
deleteMarker: BigInt(0),
},
});
}
if (!source) {
return resTarget;
}
const { usedCapacity, objectCount, accountOwnerID } = source;
resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : 0;
resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : 0;
resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : 0;
resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : 0;
resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : 0;
resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : 0;
resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : 0;
resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : 0;
resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : 0;
resTarget.usedCapacity.current += usedCapacity && usedCapacity.current ? usedCapacity.current : BigInt(0);
resTarget.usedCapacity.nonCurrent += usedCapacity && usedCapacity.nonCurrent ? usedCapacity.nonCurrent : BigInt(0);
resTarget.usedCapacity._currentCold += usedCapacity && usedCapacity._currentCold ? usedCapacity._currentCold : BigInt(0);
resTarget.usedCapacity._nonCurrentCold += usedCapacity && usedCapacity._nonCurrentCold ? usedCapacity._nonCurrentCold : BigInt(0);
resTarget.usedCapacity._currentRestoring += usedCapacity && usedCapacity._currentRestoring ? usedCapacity._currentRestoring : BigInt(0);
resTarget.usedCapacity._currentRestored += usedCapacity && usedCapacity._currentRestored ? usedCapacity._currentRestored : BigInt(0);
resTarget.usedCapacity._nonCurrentRestoring += usedCapacity && usedCapacity._nonCurrentRestoring ? usedCapacity._nonCurrentRestoring : BigInt(0);
resTarget.usedCapacity._nonCurrentRestored += usedCapacity && usedCapacity._nonCurrentRestored ? usedCapacity._nonCurrentRestored : BigInt(0);
resTarget.usedCapacity._incompleteMPUParts += usedCapacity && usedCapacity._incompleteMPUParts ? usedCapacity._incompleteMPUParts : BigInt(0);

resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : 0;
resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : 0;
resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : 0;
resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : 0;
resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : 0;
resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : 0;
resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : 0;
resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : 0;
resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : 0;
resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : 0;
resTarget.objectCount.current += objectCount && objectCount.current ? objectCount.current : BigInt(0);
resTarget.objectCount.nonCurrent += objectCount && objectCount.nonCurrent ? objectCount.nonCurrent : BigInt(0);
resTarget.objectCount.deleteMarker += objectCount && objectCount.deleteMarker ? objectCount.deleteMarker : BigInt(0);
resTarget.objectCount._currentCold += objectCount && objectCount._currentCold ? objectCount._currentCold : BigInt(0);
resTarget.objectCount._nonCurrentCold += objectCount && objectCount._nonCurrentCold ? objectCount._nonCurrentCold : BigInt(0);
resTarget.objectCount._currentRestoring += objectCount && objectCount._currentRestoring ? objectCount._currentRestoring : BigInt(0);
resTarget.objectCount._currentRestored += objectCount && objectCount._currentRestored ? objectCount._currentRestored : BigInt(0);
resTarget.objectCount._nonCurrentRestoring += objectCount && objectCount._nonCurrentRestoring ? objectCount._nonCurrentRestoring : BigInt(0);
resTarget.objectCount._nonCurrentRestored += objectCount && objectCount._nonCurrentRestored ? objectCount._nonCurrentRestored : BigInt(0);
resTarget.objectCount._incompleteMPUUploads += objectCount && objectCount._incompleteMPUUploads ? objectCount._incompleteMPUUploads : BigInt(0);

resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : 0;
resTarget.usedCapacity._inflightsPreScan += usedCapacity && usedCapacity._inflightsPreScan ? usedCapacity._inflightsPreScan : BigInt(0);
if (accountOwnerID) {
resTarget.accountOwnerID = accountOwnerID;
}

resTarget.usedCapacity.current += usedCapacity
? usedCapacity._currentCold + usedCapacity._currentRestored + usedCapacity._currentRestoring
+ usedCapacity._incompleteMPUParts : 0;
+ usedCapacity._incompleteMPUParts : BigInt(0);
resTarget.usedCapacity.nonCurrent += usedCapacity
? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : 0;
? usedCapacity._nonCurrentCold + usedCapacity._nonCurrentRestored + usedCapacity._nonCurrentRestoring : BigInt(0);
resTarget.objectCount.current += objectCount
? objectCount._currentCold + objectCount._currentRestored + objectCount._currentRestoring
+ objectCount._incompleteMPUUploads : 0;
+ objectCount._incompleteMPUUploads : BigInt(0);
resTarget.objectCount.nonCurrent += objectCount
? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : 0;
? objectCount._nonCurrentCold + objectCount._nonCurrentRestored + objectCount._nonCurrentRestoring : BigInt(0);

return resTarget;
}
Expand Down
31 changes: 21 additions & 10 deletions DataReport/collectBucketMetricsAndUpdateBucketCapacityInfo.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ function isSOSCapacityInfoEnabled(bucketInfo) {
function isValidBucketStorageMetrics(bucketMetric) {
return bucketMetric
&& bucketMetric.usedCapacity
&& typeof bucketMetric.usedCapacity.current === 'number'
&& typeof bucketMetric.usedCapacity.nonCurrent === 'number'
&& bucketMetric.usedCapacity.current > -1
&& bucketMetric.usedCapacity.nonCurrent > -1;
&& typeof bucketMetric.usedCapacity.current === 'bigint'
&& typeof bucketMetric.usedCapacity.nonCurrent === 'bigint'
&& bucketMetric.usedCapacity.current > BigInt(-1)
&& bucketMetric.usedCapacity.nonCurrent > BigInt(-1);
}

function isValidCapacityValue(capacity) {
return (Number.isSafeInteger(capacity) && capacity >= 0);
return (typeof capacity === 'bigint' && capacity >= BigInt(0));
}

function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callback) {
Expand Down Expand Up @@ -74,22 +74,33 @@ function collectBucketMetricsAndUpdateBucketCapacityInfo(mongoClient, log, callb
return nxt(null, doc);
}),
(storageMetricDoc, nxt) => {
let bucketStorageUsed = -1;
let bucketStorageUsed = BigInt(-1);
console.log('storageMetricDoc', storageMetricDoc);
if (isValidBucketStorageMetrics(storageMetricDoc)) {
console.log(' here in the isValidBucketStorageMetrics');
// Do not count the objects in cold for SOSAPI
bucketStorageUsed = storageMetricDoc.usedCapacity.current
+ storageMetricDoc.usedCapacity.nonCurrent;
console.log('bucketStorageUsed', { bucketStorageUsed });
}
// read Capacity from bucket._capabilities
const { Capacity } = bucket.getCapabilities().VeeamSOSApi.CapacityInfo;
console.log('Capacity', Capacity);

let available = -1;
let capacity = -1;
if (isValidCapacityValue(Capacity)) { // is Capacity value is valid
let available = BigInt(-1);
let capacity = BigInt(-1);
if (isValidCapacityValue(Capacity)) {
console.log('we are in the valid capacity value');
// is Capacity value is valid
capacity = Capacity;
// if bucket storage used is valid and capacity is bigger than used
if (bucketStorageUsed !== -1 && (capacity - bucketStorageUsed) >= 0) {
if (bucketStorageUsed !== BigInt(-1) && (capacity - bucketStorageUsed) >= BigInt(0)) {
console.log('WE ARE IN THE CONDITION');
available = capacity - bucketStorageUsed;
console.log('available', available);
console.log('capacity', typeof capacity);
console.log('bucketStorageUsed', typeof bucketStorageUsed);
console.log('available', typeof available);
}
}
return mongoClient.updateBucketCapacityInfo(bucketName, {
Expand Down
16 changes: 8 additions & 8 deletions bucketVersionsStats.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,12 +132,12 @@ const s3 = new AWS.S3(Object.assign(options, s3Options));

const stats = {
current: {
count: 0,
size: 0,
count: BigInt(0),
size: BigInt(0),
},
noncurrent: {
count: 0,
size: 0,
count: BigInt(0),
size: BigInt(0),
},
};

Expand All @@ -147,8 +147,8 @@ let VersionIdMarker;
function _logProgress(message) {
const loggedStats = {
total: {
count: stats.current.count + stats.noncurrent.count,
size: stats.current.size + stats.noncurrent.size,
count: BigInt(stats.current.count + stats.noncurrent.count),
size: BigInt(stats.current.size + stats.noncurrent.size),
},
...stats,
};
Expand Down Expand Up @@ -199,8 +199,8 @@ function listBucket(bucket, cb) {
}
}
const statObj = version.IsLatest ? stats.current : stats.noncurrent;
statObj.count += 1;
statObj.size += version.Size || 0;
statObj.count += BigInt(1);
statObj.size += version.Size || BigInt(0);
if (VERBOSE) {
log.info('version info', {
bucket: BUCKET,
Expand Down
4 changes: 2 additions & 2 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"scripts": {
"lint": "eslint $(git ls-files '*.js')",
"test:unit": "LOCATION_CONFIG_FILE='tests/conf/locationConfig.json' yarn jest --verbose --logHeapUsage --projects jest.config.js --coverage --testPathPattern='tests/unit/[\\w/-]+\\.[tj]s'",
"test:functional": "LOCATION_CONFIG_FILE='tests/conf/locationConfig.json' yarn jest --verbose --logHeapUsage --projects jest.config.js --coverage --testPathPattern='tests/functional/[\\w/-]+\\.[tj]s'"
"test:functional": "LOCATION_CONFIG_FILE='tests/conf/locationConfig.json' yarn jest --verbose --logHeapUsage --projects jest.config.js --coverage --testPathPattern='tests/functional/collectBucketMetricsAndUpdateBucketCapacityInfo.js'"
},
"repository": {
"type": "git",
Expand All @@ -28,7 +28,7 @@
"dependencies": {
"@senx/warp10": "^1.1.2",
"JSONStream": "^1.3.5",
"arsenal": "git+https://github.com/scality/arsenal#8.1.142",
"arsenal": "git+https://github.com/scality/arsenal#3c8219beb3785a66d1a5d74b4dff14eda869348f",
"async": "^2.6.4",
"aws-sdk": "^2.1005.0",
"bucketclient": "git+https://github.com/scality/bucketclient#8.1.5",
Expand Down
8 changes: 4 additions & 4 deletions service-level-sidecar/report.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,14 @@ const { getAccountIdForCanonicalId } = require('./vault');
class MetricReport {
constructor(factor = 1) {
this.factor = factor;
this.count = 0;
this.bytes = 0;
this.count = BigInt(0);
this.bytes = BigInt(0);
}

/**
* Add the given data to the metric report
* @param {integer} count - Amount to increment object count
* @param {integer} bytes - Amount to increment bytes stored
* @param {bigint} count - Amount to increment object count
* @param {bigint} bytes - Amount to increment bytes stored
* @returns {undefined} -
*/
update(count, bytes) {
Expand Down
Loading

0 comments on commit 9325b3f

Please sign in to comment.