youtbe parser

.github/workflows/main.yml

@@ -18,26 +18,10 @@ permissions:
 env:
   PLUGIN_JSON: "0.0.1"
   TAG_EXISTS: false
-  PLUGIN_NAME: "my_plugin"
+  PLUGIN_NAME: "catube"
 
 jobs:
-# This will be deleted by setup.py
-  check:
-    runs-on: ubuntu-latest
-    outputs:
-      plugin_name: ${{ steps.init.outputs.plugin_name }}
-    steps:
-    - name: Get plugin name
-      id: init
-      run: |
-        echo "plugin_name=${{ env.PLUGIN_NAME }}" >> $GITHUB_OUTPUT
-
-# This is the end of the removed section
   release:
-  # This will be deleted by setup.py
-    needs: check
-    if: startsWith(needs.check.outputs.plugin_name, 'MY_PLUGIN') == false
-  # This is the end of the removed section
     runs-on: ubuntu-latest
     steps:
       - name: Checkout

README.md

@@ -1,4 +1,4 @@
-# My plugin
+# Catube
 
 [![awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=awesome+plugin&color=383938&style=for-the-badge&logo=cheshire_cat_ai)](https://)  
 [![Awesome plugin](https://custom-icon-badges.demolab.com/static/v1?label=&message=Awesome+plugin&color=000000&style=for-the-badge&logo=cheshire_cat_ai)](https://)  

catube.py

@@ -0,0 +1,38 @@
+import re
+from abc import ABC
+from typing import Iterator
+
+from pytube import extract
+from youtube_transcript_api import YouTubeTranscriptApi
+from youtube_transcript_api.formatters import TextFormatter, JSONFormatter
+from langchain.docstore.document import Document
+from langchain.document_loaders.base import BaseBlobParser
+from langchain.document_loaders.blob_loaders.schema import Blob
+
+from cat.mad_hatter.decorators import hook
+
+
+def parse_youtube_url(url: str) -> str:
+    data = re.findall(r"(?:v=|\/)([0-9A-Za-z_-]{11}).*", url)
+    if data:
+        return data[0]
+    return ""
+
+
+class YoutubeParser(BaseBlobParser, ABC):
+    def __init__(self):
+        self.formatter = TextFormatter()
+
+    def lazy_parse(self, blob: Blob) -> Iterator[Document]:
+        video_id = extract.video_id(blob.source)
+
+        transcript = YouTubeTranscriptApi.get_transcripts([video_id], languages=["en", "it"], preserve_formatting=True)
+        text = self.formatter.format_transcript(transcript[0][video_id])
+
+        yield Document(page_content=text, metadata={})
+
+
+@hook
+def rabbithole_instantiates_parsers(file_handlers: dict, cat) -> dict:
+    file_handlers["video/mp4"] = YoutubeParser()
+    return file_handlers

plugin.json

@@ -1,10 +1,10 @@
 {
-    "name": "My plugin",
+    "name": "Catube",
     "version": "0.0.1",
-    "description": "Description of my_plugin.",
-    "author_name": "Me",
-    "author_url": "https://mywebsite.me",
-    "plugin_url": "https://github.com/my_name/my_plugin",
-    "tags": "cat, template, example",
-    "thumb": "https://raw.githubusercontent.com/my_repo_path/my_plugin.png"
+    "description": "Description of catube.",
+    "author_name": "Nicola Corbellini",
+    "author_url": "",
+    "plugin_url": "https://github.com/Furrmidable-Crew/CaTube",
+    "tags": "Cat, RabbitHole, DocumentsLoading",
+    "thumb": "https://raw.githubusercontent.com/my_repo_path/catube.png"
 }

requirements.txt

@@ -0,0 +1,2 @@
+youtube-transcript-api
+pytube