fetchData.js

import fetch from "node-fetch";
import { search } from "fast-fuzzy";
import { htmlToText } from "html-to-text";
import { filterObjectArray } from "./utils/arrayUtils.js";
import config from "./config.js";
import { dataRetrievalConstants } from "./constants/dataRetrieval.js";

const {
  IDC_API_BASE_URL,
  IDC_COLLECTION_BASE_URL,
  IDC_API_COLLECTIONS_ENDPOINT,
  TCIA_API_BASE_URL,
  TCIA_COLLECTION_BASE_URL,
  TCIA_API_COLLECTIONS_ENDPOINT,
  TCIA_API_SERIES_ENDPOINT,
} = dataRetrievalConstants;

/**
 * Retrieves image collection data from the IDC API and filters for collections relevant to ICDC.
 *
 * @async
 * @returns {Promise<string[]>} - Promise that resolves with an array of IDC collections.
 * @throws {Error} - Throws error if there is an issue with the IDC response.
 */
async function getIdcCollections() {
  try {
    const response = await fetch(
      `${IDC_API_BASE_URL}${IDC_API_COLLECTIONS_ENDPOINT}`
    );
    if (!response.ok) {
      throw new Error(
        `IDC collection request failed (${response.status}): ${response.statusText}`
      );
    }
    const data = await response.json();
    return filterObjectArray(data["collections"], "collection_id", "icdc_");
  } catch (error) {
    console.error("Error fetching IDC collections: ", error.message);
    throw new Error(`IDC internal server error: ${error.message}`);
  }
}

/**
 * Retrieves image collection data from the TCIA API and filters for collection IDs relevant to ICDC.
 *
 * @async
 * @returns {Promise<string[]>} - Promise that resolves with an array of TCIA collection IDs.
 * @throws {Error} - Throws error if there is an issue with the TCIA response.
 */
async function getTciaCollections() {
  try {
    const response = await fetch(
      `${TCIA_API_BASE_URL}${TCIA_API_COLLECTIONS_ENDPOINT}`
    );
    if (!response.ok) {
      throw new Error(
        `TCIA collection request failed (${response.status}): ${response.statusText}`
      );
    }
    const data = await response.json();
    return filterObjectArray(data, "Collection", "ICDC-").map(
      (obj) => obj.Collection
    );
  } catch (error) {
    console.error("Error fetching TCIA collections: ", error.message);
    throw new Error(`TCIA internal server error: ${error.message}`);
  }
}

/**
 * Retrieves data from TCIA API for a specific TCIA image collection.
 *
 * @async
 * @param {string} collection_id - ID of TCIA image collection.
 * @returns {Promise<Object>} - Promise that resolves with data for specified TCIA collection.
 * @throws {Error} - Throws error if there is an issue with the TCIA response.
 */
async function getTciaCollectionData(collection_id) {
  try {
    const response = await fetch(
      `${TCIA_API_BASE_URL}${TCIA_API_SERIES_ENDPOINT}${collection_id}`
    );
    if (!response.ok) {
      throw new Error(
        `TCIA collection request failed (${response.status}): ${response.statusText}`
      );
    }
    return await response.json();
  } catch (error) {
    console.error("Error fetching TCIA collection data: ", error.message);
    throw new Error(`TCIA internal server error: ${error.message}`);
  }
}

/**
 * Retrieves study data from the ICDC backend via a GraphQL query.
 *
 * @async
 * @returns {Promise<Object[]>} - Promise that resolves with an array of ICDC study data objects.
 * @throws {Error} - Throws error if there is an issue with the ICDC response.
 */
async function getIcdcStudyData() {
  try {
    const body = JSON.stringify({
      query: `{
              studiesByProgram {
                  clinical_study_designation
                  numberOfImageCollections
                  numberOfCRDCNodes
              }
      }`,
    });
    const response = await fetch(config.BENTO_BACKEND_GRAPHQL_URI, {
      method: "POST",
      body: body,
    });
    if (!response.ok) {
      throw new Error(
        `ICDC studies request failed (${response.status}): ${response.statusText}`
      );
    }
    const responseJson = await response.json();
    if (!responseJson?.data?.studiesByProgram) {
      throw new Error("ICDC response missing required data");
    }
    return responseJson.data.studiesByProgram;
  } catch (error) {
    console.error("Error fetching ICDC study data: ", error.message);
    throw new Error(`ICDC internal server error: ${error.message}`);
  }
}

/**
 * Iterates a list of TCIA collection names and gets corresponding metadata for each collection.
 *
 * @async
 * @param {string[]} tciaCollections - Array of TCIA collection names.
 * @returns {Object} - Object containing collection data.
 */
async function getTciaCollectionsData(tciaCollections) {
  const results = await Promise.all(
    tciaCollections.map(async (collection) => {
      const data = await getTciaCollectionData(collection);
      return { [collection]: data };
    })
  );
  return Object.assign({}, ...results);
}

/**
 * Maps collection metadata to a specified IDC collection.
 *
 * @param {string} collectionId - IDC collection name.
 * @param {Object[]} idcCollections - An array of IDC collection data objects.
 * @param {string} icdcStudy - ICDC study name.
 * @returns {Object} - Object containing metadata for specified IDC collection.
 */
function getIdcCollectionMetadata(collectionId, idcCollections, icdcStudy) {
  let idcCollectionMetadata = idcCollections.find(
    (obj) => obj.collection_id === collectionId
  );
  // handle oddly-formatted response HTML for GLIOMA01
  const cleanedDescText = htmlToText(idcCollectionMetadata["description"], {
    wordwrap: null,
  });
  if (icdcStudy.clinical_study_designation === "GLIOMA01") {
    idcCollectionMetadata["description"] = cleanedDescText
      .replace(/\n\n|\s*\[.*?\]\s*/g, " ")
      .replace(/ \./g, ".")
      .replace(" ICDC-Glioma", "");
  } else {
    idcCollectionMetadata["description"] = cleanedDescText;
  }
  return idcCollectionMetadata;
}

/**
 * Maps collection metadata to a specified TCIA collection.
 *
 * @param {string} collectionId - TCIA collection name.
 * @param {Object[]} tciaCollectionsData - Object containing data for TCIA collections.
 * @param {string} icdcStudy - ICDC study name.
 * @returns {Object} - Object containing metadata for specified TCIA collection.
 */
function getTciaCollectionMetadata(
  collectionId,
  tciaCollectionsData,
  icdcStudy
) {
  let tciaCollectionMetadata = tciaCollectionsData[collectionId];
  let totalImages = tciaCollectionMetadata.reduce(
    (tot, obj) => tot + parseInt(obj.ImageCount),
    0
  );
  const totalPatients = [
    ...new Set(tciaCollectionMetadata.map((obj) => obj.PatientID)),
  ].length;
  const uniqueModalities = [
    ...new Set(tciaCollectionMetadata.map((obj) => obj.Modality)),
  ];
  const uniqueBodypartsExamined = [
    ...new Set(tciaCollectionMetadata.map((obj) => obj.BodyPartExamined)),
  ];
  // hardcode inaccessible TCIA data for GLIOMA01
  if (icdcStudy.clinical_study_designation === "GLIOMA01") {
    uniqueModalities.push("Histopathology");
    totalImages += 84;
  }
  return {
    Collection: collectionId,
    Aggregate_PatientID: totalPatients,
    Aggregate_Modality: uniqueModalities,
    Aggregate_BodyPartExamined: uniqueBodypartsExamined,
    Aggregate_ImageCount: totalImages,
  };
}

/**
 * Matches any ICDC-relevant external data to specific ICDC study.
 *
 * @async
 * @param {string} icdcStudy - ICDC study name.
 * @param {Object[]} idcCollections - Array of IDC collection data objects.
 * @param {string[]} tciaCollections - Array of TCIA collection names.
 * @param {string} tciaCollectionsData - Object containing data for TCIA collections.
 * @returns {Promise<Object[]>} - Promise that resolves with array of data collection objects matched to corresponding ICDC study.
 */
async function mapMatchesToStudy(
  icdcStudy,
  idcCollections,
  tciaCollections,
  tciaCollectionsData
) {
  let collectionUrls = [];

  // fuzzy match strings using damerau-levenshtein distance
  let idcMatches = search(
    icdcStudy.clinical_study_designation,
    idcCollections.map((obj) => obj.collection_id)
  );
  let tciaMatches = search(
    icdcStudy.clinical_study_designation,
    tciaCollections
  );

  if (idcMatches.length !== 0) {
    const idcResults = await Promise.all(
      idcMatches.map(async (match) => {
        const idcCollectionUrl = `${IDC_COLLECTION_BASE_URL}${match}`;
        const idcCollectionMetadata = await getIdcCollectionMetadata(
          match,
          idcCollections,
          icdcStudy
        );
        return {
          repository: "IDC",
          url: idcCollectionUrl,
          metadata: idcCollectionMetadata,
        };
      })
    );
    collectionUrls.push(...idcResults);
  }
  if (tciaMatches.length !== 0) {
    const tciaResults = await Promise.all(
      tciaMatches
        .filter((match) => tciaCollectionsData[match]?.length > 0)
        .map(async (match) => {
          const tciaCollectionUrl = `${TCIA_COLLECTION_BASE_URL}${match}`;
          let tciaCollectionMetadata = await getTciaCollectionMetadata(
            match,
            tciaCollectionsData,
            icdcStudy
          );
          return {
            repository: "TCIA",
            url: tciaCollectionUrl,
            metadata: tciaCollectionMetadata,
          };
        })
    );
    collectionUrls.push(...tciaResults);
  }
  return collectionUrls;
}

/**
 * Collects/assembles external data collection metadata and counts for corresponding ICDC studies.
 *
 * @async
 * @param {string[]} icdcStudies - Array of ICDC study names.
 * @param {Object[]} idcCollections - Array of IDC collection data objects.
 * @param {string[]} tciaCollections - Array of TCIA collection names.
 * @param {Object[]} tciaCollectionsData - Object containing data for TCIA collections.
 * @returns {Promise<Object[]>} - Promise that resolves with an array of external data collection mappings to relevant ICDC studies.
 */
async function collectMappings(
  icdcStudies,
  idcCollections,
  tciaCollections,
  tciaCollectionsData
) {
  const mappings = await Promise.all(
    icdcStudies.map(async (study) => {
      const collectionUrls = await mapMatchesToStudy(
        study,
        idcCollections,
        tciaCollections,
        tciaCollectionsData
      );
      if (study?.numberOfCRDCNodes > 0) {
        return {
          CRDCLinks: collectionUrls,
          numberOfCRDCNodes: study?.numberOfCRDCNodes,
          numberOfImageCollections: study?.numberOfImageCollections,
          clinical_study_designation: study?.clinical_study_designation,
        };
      }
      return null;
    })
  );
  return mappings.filter(Boolean);
}

/**
 * Maps ICDC-related data from external APIs to corresponding ICDC studies.
 *
 * @async
 * @returns {Promise<Object[]>} - Promise that resolves with an array of data collection mappings.
 */
async function mapExternalDataToStudies() {
  try {
    const [icdcStudies, idcCollections, tciaCollections] = await Promise.all([
      getIcdcStudyData(),
      getIdcCollections(),
      getTciaCollections(),
    ]);
    const tciaCollectionsData = await getTciaCollectionsData(tciaCollections);
    const collectionMappings = await collectMappings(
      icdcStudies,
      idcCollections,
      tciaCollections,
      tciaCollectionsData
    );
    return collectionMappings;
  } catch (error) {
    console.error(error);
    return error;
  }
}

export {
  getIdcCollections,
  getTciaCollections,
  getTciaCollectionData,
  getTciaCollectionsData,
  getIcdcStudyData,
  getIdcCollectionMetadata,
  getTciaCollectionMetadata,
  mapMatchesToStudy,
  collectMappings,
  mapExternalDataToStudies,
};