llm-semantic-annotator.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dKquLaEdEeVv"
      },
      "source": [
        "# LLMSemanticAnnotator: Advanced Semantic Annotation for Plant Biology Research\n",
        "\n",
        "Installation de la lib"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "vscode": {
          "languageId": "shellscript"
        }
      },
      "outputs": [],
      "source": [
        "# JEAN-ZAY\n",
        "if [ -n \"$WORK\" ]; then\n",
        "    !export PYTHONUSERBASE=$WORK/python_base\n",
        "    !export HF_HOME=$WORK/hg_cache/huggingface\n",
        "fi"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tKZS68aVEQbW",
        "outputId": "d8713e6c-a469-477c-b29b-99368362cf17"
      },
      "outputs": [],
      "source": [
        "\n",
        "#!pip uninstall llm_semantic_annotator\n",
        "!pip install git+https://github.com/p2m2/encoder-ontology-match-abstract\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V6uHhs_Bz5B_"
      },
      "source": [
        "Creation du fichier de configuration pour l'alignement des ontologies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ccvilfdTFYl2"
      },
      "outputs": [],
      "source": [
        "import json\n",
        "\n",
        "data = {\n",
        "    \"encodeur\" : \"sentence-transformers/all-MiniLM-L6-v2\",\n",
        "    \"threshold_similarity_tag_chunk\" : 0.70,\n",
        "    \"threshold_similarity_tag\" : 0.80,\n",
        "    \"batch_size\" : 32,\n",
        "\n",
        "    \"populate_owl_tag_embeddings\" : {\n",
        "        \"prefix\" : {\n",
        "            \"rdf\" : \"http://www.w3.org/1999/02/22-rdf-syntax-ns#\",\n",
        "            \"rdfs\" : \"http://www.w3.org/2000/01/rdf-schema#\",\n",
        "            \"owl\" : \"http://www.w3.org/2002/07/owl#\"\n",
        "        },\n",
        "        \"ontologies\": {\n",
        "            \"planteome_link\" : {\n",
        "                \"peco\": {\n",
        "                    \"url\": \"http://purl.obolibrary.org/obo/peco.owl\",\n",
        "                    \"prefix\": \"http://purl.obolibrary.org/obo/PECO_\",\n",
        "                    \"format\": \"xml\",    \n",
        "                    \"label\" : \"rdfs:label\",\n",
        "                    \"properties\": [\"obo:IAO_0000115\",\"rdfs:comment\",\"owl:annotatedTarget\"]\n",
        "                },\n",
        "                \"po\": {\n",
        "                    \"url\": \"http://purl.obolibrary.org/obo/po.owl\",\n",
        "                    \"prefix\": \"http://purl.obolibrary.org/obo/PO_\",\n",
        "                    \"format\": \"xml\",\n",
        "                    \"label\" : \"rdfs:label\",\n",
        "                    \"properties\": [\"obo:IAO_0000115\",\"rdfs:comment\",\"owl:annotatedTarget\"]\n",
        "                },\n",
        "                \"pso\": {\n",
        "                    \"url\": \"http://purl.obolibrary.org/obo/pso.owl\",\n",
        "                    \"prefix\": \"http://purl.obolibrary.org/obo/PSO_\",\n",
        "                    \"format\": \"xml\",\n",
        "                    \"label\" : \"rdfs:label\",\n",
        "                    \"properties\": [\"obo:IAO_0000115\",\"rdfs:comment\",\"owl:annotatedTarget\"]\n",
        "                },\n",
        "                \"to\": {\n",
        "                    \"url\": \"http://purl.obolibrary.org/obo/to.owl\",\n",
        "                    \"prefix\": \"http://purl.obolibrary.org/obo/TO_\",\n",
        "                    \"format\": \"xml\",\n",
        "                    \"label\" : \"rdfs:label\",\n",
        "                    \"properties\": [\"obo:IAO_0000115\",\"rdfs:comment\",\"owl:annotatedTarget\"]\n",
        "                }\n",
        "            }\n",
        "        }\n",
        "    },\n",
        "    \"populate_abstract_embeddings\" : {\n",
        "        \"abstracts_per_file\" : 500,\n",
        "        \"from_ncbi_api\" : {\n",
        "            \"ncbi_api_chunk_size\" : 200,\n",
        "            \"debug_nb_ncbi_request\" : -1,\n",
        "            \"retmax\" : 2000,\n",
        "            \"selected_term\" : [\n",
        "                \"Crops%2C+Agricultural%2Fmetabolism%5BMeSH%5D\"\n",
        "            ]\n",
        "        }\n",
        "    }\n",
        "}\n",
        "\n",
        "\n",
        "with open('config.json', 'w') as fichier:\n",
        "    json.dump(data, fichier, indent=4)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Populate OWL tag embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yJSBbOQr0Fj4",
        "outputId": "8d7bcec6-a612-426c-a305-ec9daba01946"
      },
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json populate_owl_tag_embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Populate abstract embeddings"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_D4-Z28z8E6a",
        "outputId": "f7967200-2131-4802-b565-306f870a95c4"
      },
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json populate_abstract_embeddings"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yfgMXvYb8Fjh"
      },
      "source": [
        "## Compute similarities between tags and abstract chunks\n",
        "------"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HFGu6xtd8e6J",
        "outputId": "62021bb9-51b0-4442-956d-8e3efc18a284"
      },
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json compute_tag_chunk_similarities"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Display similarities information"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json display_summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Build turtle knowledge graph"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json build_rdf_graph"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Build dataset abstracts annotations CSV file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "!llm-semantic-annotator config.json build_dataset_abstracts_annotations"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 329
        },
        "id": "d7GB-u4C3eyz",
        "outputId": "8e08fa54-9819-4bbc-f737-7ae7cb2494c3"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import tarfile\n",
        "from google.colab import files\n",
        "\n",
        "def compress_and_download(directory_name):\n",
        "    # Nom de l'archive\n",
        "    archive_name = f\"{directory_name}.tar.gz\"\n",
        "\n",
        "    # Compression du répertoire\n",
        "    with tarfile.open(archive_name, \"w:gz\") as tar:\n",
        "        tar.add(directory_name, arcname=os.path.basename(directory_name))\n",
        "\n",
        "    print(f\"Compression terminée : {archive_name}\")\n",
        "\n",
        "    # Vérification de l'existence du fichier compressé\n",
        "    if os.path.exists(archive_name):\n",
        "        # Téléchargement du fichier\n",
        "        files.download(archive_name)\n",
        "        print(f\"Le fichier {archive_name} a été téléchargé.\")\n",
        "    else:\n",
        "        print(f\"Erreur : Le fichier {archive_name} n'a pas été créé.\")\n",
        "\n",
        "# Utilisation de la fonction\n",
        "directory_to_compress = \"config_workdir\"\n",
        "compress_and_download(directory_to_compress)\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "gpuType": "V28",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}