diff --git a/src/config/index.ts b/src/config/index.ts index 9651e25..9538e26 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -18,7 +18,9 @@ export const configs = { maxFileSize: BigInt(getEnvOrExit("TASK_MAX_FILE_SIZE", "10485760", false)), providerMnemonic: getEnvOrExit("TASK_PROVIDER_MNEMONIC", ""), providerMinBalance: getEnvOrExit("TASK_PROVIDER_MIN_BALANCE", "1000000000"), - submitStorageProofBefore: getEnvOrExit("TASK_SUBMIT_STORAGE_PROOF_BEFORE", "1800") + submitStorageProofBefore: getEnvOrExit("TASK_SUBMIT_STORAGE_PROOF_BEFORE", "1800"), + maxDownloadHeaderTimes: getEnvOrExit("TASK_MAX_DOWNLOAD_HEADER_TIMES", "10"), + maxDownloadChildTimes: getEnvOrExit("TASK_MAX_DOWNLOAD_CHILD_TIMES", "10"), }, tonStorageUtilsApi: getEnvOrExit('TON_STORAGE_UTILS_API', 'http://localhost:8192'), } diff --git a/src/dao/task.ts b/src/dao/task.ts index 410951d..2be36ab 100644 --- a/src/dao/task.ts +++ b/src/dao/task.ts @@ -45,7 +45,17 @@ export class Task { type: DataTypes.BIGINT, allowNull: false, defaultValue: 0 - } + }, + download_header_retry_times: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0 + }, + download_child_retry_times: { + type: DataTypes.INTEGER, + allowNull: false, + defaultValue: 0 + }, }, { indexes: [ { diff --git a/src/merkle/tonsutils.ts b/src/merkle/tonsutils.ts index dd7f29d..b81930b 100644 --- a/src/merkle/tonsutils.ts +++ b/src/merkle/tonsutils.ts @@ -41,8 +41,20 @@ export function sleep(time: number) { const baseUrl = configs.tonStorageUtilsApi export async function getTonBagDetails(bag_id: string) { return fetch(`${baseUrl}/api/v1/details?bag_id=${bag_id}`) - .then((res) => res.json()) - .then((item) => item as BagDetail); + .then((res) => { + if (res.status == 200 || res.status == 404) { + return res.json() + } else { + throw new Error("Call storage api failed") + } + }) + .then((item) => { + const bagId = item.bag_id + if (bagId === undefined) { + return null; + } + return item as BagDetail; + }); } export async function addTonBag({ @@ -67,29 +79,6 @@ export async function addTonBag({ }); } -export async function downloadTonBag(bag_id: string, waitCompleted: boolean = false) { - await addTonBag({ bag_id }); - let bd: BagDetail; - // check header - while (true) { - await sleep(1); - bd = await getTonBagDetails(bag_id); - if (bd.header_loaded) { - break; - } - } - // down all - await addTonBag({ bag_id, files: bd.files.map((f) => f.index), donwload_all: true }); - // check all - while (waitCompleted) { - await sleep(2000); - bd = await getTonBagDetails(bag_id); - if (bd.downloaded == bd.size) { - return true; - } - } - return true -} export async function downloadChildTonBag(bag_id: string) { const bd = await getTonBagDetails(bag_id); @@ -100,10 +89,16 @@ export async function downloadChildTonBag(bag_id: string) { export async function downloadTonBagSuccess(bag_id: string): Promise { const bd = await getTonBagDetails(bag_id); - return bd.downloaded == bd.size; + if (bd) { + return bd.downloaded == bd.size; + } + return false; } export async function downloadHeaderSuccess(bag_id: string): Promise { const bd = await getTonBagDetails(bag_id); - return bd.header_loaded; + if (bd) { + return bd.header_loaded; + } + return false; } diff --git a/src/service/storage.ts b/src/service/storage.ts index a97419d..cc4149b 100644 --- a/src/service/storage.ts +++ b/src/service/storage.ts @@ -11,6 +11,7 @@ import { } from "../merkle/tonsutils"; import {logger} from "../util/logger"; import {Op} from "sequelize"; +import {configs} from "../config"; export async function downloadTorrentHeaders() { while(true) { @@ -39,7 +40,22 @@ export async function downloadTorrentHeaders() { } await addTonBag(torrentHash); } catch (e) { - logger.error(`Failed to download meta bag ${torrentHash}: ${e}`); + logger.error(`Failed to download meta bag ${torrentHash}: ${e.message}`); + const retryTimes = task.download_header_retry_times + 1; + let item: any = { + download_header_retry_times: retryTimes + } + if (retryTimes > Number(configs.task.maxDownloadHeaderTimes)) { + item = { + ...item, + task_state: TaskState.download_torrent_header_failed + } + } + await Task.model.update(item, { + where: { + id: task.id + } + }); continue; } await Task.model.update({ @@ -55,6 +71,9 @@ export async function downloadTorrentHeaders() { async function checkDownloadState(torrentHash: string, task: any){ const bagDetail = await getTonBagDetails(torrentHash); + if (bagDetail == null) { + return false; + } if (bagDetail.downloaded === bagDetail.size) { await Task.model.update({ task_state: TaskState.download_torrent_success @@ -103,7 +122,22 @@ export async function downloadChildFiles() { } await downloadChildTonBag(torrentHash); } catch (e) { - logger.error(`Failed to download child bag ${torrentHash}: ${e}`); + logger.error(`Failed to download child bag ${torrentHash}: ${e.message}`); + const retryTimes = task.download_child_retry_times + 1; + let item: any = { + download_child_retry_times: retryTimes + } + if (retryTimes > Number(configs.task.maxDownloadChildTimes)) { + item = { + ...item, + task_state: TaskState.download_torrent_child_file_failed + } + } + await Task.model.update(item, { + where: { + id: task.id + } + }); continue; } await Task.model.update({ diff --git a/src/service/task.ts b/src/service/task.ts index 69c1121..1aaf9d0 100644 --- a/src/service/task.ts +++ b/src/service/task.ts @@ -8,9 +8,6 @@ import {sleep} from "../util/common"; import {sequelize} from "../db"; import {Task} from "../dao/task"; import {LAST_TASK_GENERATE_ORDER_ID} from "./constants"; -import {Address, OpenedContract} from "@ton/ton"; -import {Cell} from "@ton/core"; -import {StorageContract} from "../wrapper/StorageContract"; export async function generateTasks() { while(true) { diff --git a/src/type/common.ts b/src/type/common.ts index a07f354..bfc29ee 100644 --- a/src/type/common.ts +++ b/src/type/common.ts @@ -22,4 +22,6 @@ export enum TaskState { period_finish = 6, task_finish = 7, more_than_max_storage_provider_count = -1, + download_torrent_header_failed = -2, + download_torrent_child_file_failed = -3, } diff --git a/src/wrapper/StorageContract.ts b/src/wrapper/StorageContract.ts index 168f04f..1aa2283 100644 --- a/src/wrapper/StorageContract.ts +++ b/src/wrapper/StorageContract.ts @@ -254,7 +254,7 @@ export class StorageContract implements Contract { const storageProvidersInfo = ds.loadRef(); const storageProvidersInfoDs = storageProvidersInfo.beginParse(); const storageProviders = storageProvidersInfoDs.loadDict(number256DictionaryKey, cellDictionaryValue); - const {max_storage_provider_count} = this.loadOrderInfo(orderInfo); + const {max_storage_provider_count} = this.loadOrderInfo(firstCell); return max_storage_provider_count - storageProviders.size; } return 0;