Add caption support with new models, vuex, and edit modal integration

contentcuration/automation/settings.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from torch.cuda import is_available as is_gpu_available
+
+DEVICE = "cuda:0" if is_gpu_available() else "cpu"
+
+
+# [TRANSCRIPTION GENERATION]
+WHISPER_MODELS = dict(
+    TINY="openai/whisper-tiny",
+    BASE="openai/whisper-base",
+    SMALL="openai/whisper-small",
+    MEDIUM="openai/whisper-medium",
+    LARGE="openai/whisper-large",
+    LARGEV2="openai/whisper-large-v2",
+)
+
+
+DEV_TRANSCRIPTION_MODEL = WHISPER_MODELS['TINY']
+TRANSCRIPTION_MODEL = WHISPER_MODELS['TINY']
+
+class WhisperTask(Enum):
+    TRANSLATE = "translate"
+    TRANSCRIBE = "transcribe"
+
+# https://huggingface.co/docs/transformers/v4.29.1/en/generation_strategies#customize-text-generation
+MAX_TOKEN_LENGTH = 448
+CHUNK_LENGTH = 10

contentcuration/automation/urls.py

@@ -0,0 +1,10 @@
+from automation.views import TranscriptionsViewSet
+from django.urls import include, path
+from rest_framework import routers
+
+automation_router = routers.DefaultRouter()
+automation_router.register(r'transcribe', TranscriptionsViewSet, basename="transcribe")
+
+urlpatterns = [
+    path("api/automation/", include(automation_router.urls), name='automation'),
+]

contentcuration/automation/views.py

@@ -1,3 +1,10 @@
-# from django.shortcuts import render
+from rest_framework.viewsets import ViewSet
+from rest_framework.response import Response
+from rest_framework.permissions import AllowAny
 
-# Create your views here.
+class TranscriptionsViewSet(ViewSet):
+    def create(self, request):
+        permission_classes = [AllowAny]
+        return Response({
+            "OK":"OK"
+        })

contentcuration/contentcuration/constants/transcription_languages.py

@@ -0,0 +1,51 @@
+# The list of supported AI languages is dynamically loaded from a JSON file.
+# You can update the supported languages for transcription by modifying the 'ai_supported_languages.json' file.
+# The script then determines the intersection of languages supported by the Kolibri project and the Whisper speech-to-text model.
+# The resulting list of language codes is stored in CAPTION_LANGUAGES for creating captions.
+# Note: To update supported languages, modify the 'ai_supported_languages.json' file.
+
+import os
+import json
+from typing import List, Dict
+
+import le_utils.resources as resources
+
+
+def _ai_supported_languages() -> Dict:
+    """Loads JSON of supported AI languages"""
+    file = os.path.join(
+        os.path.dirname(os.path.realpath(__file__)),
+        "../static/ai_supported_languages.json",
+    )
+    with open(file) as f:
+        data = json.load(f)
+    return data
+
+
+WHISPER_LANGUAGES = _ai_supported_languages()
+
+
+def _load_kolibri_languages() -> List[str]:
+    """Loads the language codes from languagelookup.json and returns them as a list."""
+    filepath = resources.__path__[0]
+    kolibri_languages = []
+    with open(f"{filepath}/languagelookup.json") as f:
+        kolibri_languages = list(json.load(f).keys())
+    return kolibri_languages
+
+
+def _load_model_languages(languages: Dict[str, str]) -> List[str]:
+    """Load languages supported by the speech-to-text model.
+    :param: languages: dict mapping language codes to language names"""
+    return list(languages.keys())
+
+
+def create_captions_languages() -> List[str]:
+    """Returns the intersection of Kolibri languages and model languages"""
+    kolibri_set = set(_load_kolibri_languages())
+    model_set = set(_load_model_languages(languages=WHISPER_LANGUAGES))
+    return list(kolibri_set.intersection(model_set))
+
+
+# list of language id's ['en', 'hi', ...]
+CAPTION_LANGUAGES = create_captions_languages()

contentcuration/contentcuration/dev_urls.py

@@ -1,5 +1,6 @@
 import urllib.parse
 
+from automation.urls import urlpatterns as automation_urlpatterns
 from django.conf import settings
 from django.contrib import admin
 from django.core.files.storage import default_storage
@@ -76,6 +77,8 @@ def file_server(request, storage_path=None):
     re_path(r"^content/(?P<storage_path>.+)$", file_server),
 ]
 
+urlpatterns += automation_urlpatterns
+
 if getattr(settings, "DEBUG_PANEL_ACTIVE", False):
 
     import debug_toolbar

contentcuration/contentcuration/frontend/channelEdit/components/CaptionsTab/CaptionsTab.vue

@@ -0,0 +1,162 @@
+<template>
+
+  <div class="captions-tab-section">
+    <div class="my-4 px-4 title">
+      <ContentNodeIcon
+        v-if="node && node.kind"
+        :kind="node.kind"
+        class="mr-1"
+      />
+      <h2
+        v-if="node && node.title"
+        class="headline mx-2 notranslate"
+        data-test="title"
+      >
+        {{ $tr('header', { fileName: node.title }) }}
+      </h2>
+    </div>
+
+    <div v-if="!isGeneratingCaptions" class="mb-2 mt-2 px-4">
+      <p class="my-2">
+        {{ $tr('captionGenerationHelpText') }}
+      </p>
+
+      <LanguageDropdown
+        ref="language"
+        v-model="selectedLanguage"
+        clearable
+        :required="true"
+        :excludeLanguages="excludedLanguages"
+      />
+
+      <VBtn
+        v-if="selectedLanguage && !isGeneratingCaptions"
+        flat
+        color="primary"
+        class="font-weight-bold ml-0"
+        @click="addCaption"
+        :disabled="isCaptionFileExisting"
+      >
+        {{ $tr('generateBtn') }}
+      </VBtn>
+    </div>
+    <!-- TODO -->
+    <!-- est. time, time elapsed can be good -->
+    <p v-if="isGeneratingCaptions">
+      {{ $tr('generating', { fileName: node.title }) }}
+      <LoadingText />
+      <br>
+    </p>
+
+  </div>
+
+</template>
+
+<script>
+
+  import { mapActions, mapGetters, mapState } from 'vuex';
+  import { CaptionFile, uuid4 } from 'shared/data/resources';
+  import { notSupportedCaptionLanguages } from 'shared/leUtils/TranscriptionLanguages';
+  import ContentNodeIcon from 'shared/views/ContentNodeIcon';
+  import LoadingText from 'shared/views/LoadingText';
+  import LanguageDropdown from 'shared/views/LanguageDropdown';
+
+  export default {
+    name: 'CaptionsTab',
+    components: {
+      ContentNodeIcon,
+      LanguageDropdown,
+      LoadingText,
+    },
+    props: {
+      nodeId: {
+        type: String,
+        required: true,
+      },
+    },
+    data() {
+      return {
+        selectedLanguage: null,
+      };
+    },
+    computed: {
+      ...mapGetters('contentNode', ['getContentNode']),
+      ...mapGetters('caption', ['isGeneratingGetter']),
+      ...mapGetters('file', ['getContentNodeFiles']),
+      ...mapState('caption', ['captionFilesMap']),
+      node() {
+        return this.getContentNode(this.nodeId);
+      },
+      excludedLanguages() {
+        // excludeLanguages requires array of ids
+        return notSupportedCaptionLanguages.map(l => l.id);
+      },
+      isGeneratingCaptions() {
+        return this.isGeneratingGetter(this.nodeId);
+      },
+      isCaptionFileExisting() {
+        const language = this.selectedLanguage;
+        const fileId = this.getLongestDurationFileId();
+        return this.checkExistingFile(this.captionFilesMap[this.nodeId], fileId, language)
+      }
+    },
+    methods: {
+      ...mapActions('caption', ['addCaptionFile']),
+      checkExistingFile(captionFilesMap, captionFileId, language) {
+        if(!captionFilesMap) return false;
+        return Object.values(captionFilesMap).some(file => {
+          return file.file_id === captionFileId && file.language === language;
+        })
+      },
+      addCaption() {
+        const id = uuid4();
+        const fileId = this.getLongestDurationFileId();
+        const language = this.selectedLanguage;
+        if (!language || !fileId) return;
+        const fileExists = this.checkExistingFile(
+          this.captionFilesMap[this.nodeId], fileId, language);
+        if(fileExists) return;
+        this.addCaptionFile({
+          id: id,
+          file_id: fileId,
+          language: language,
+          nodeId: this.nodeId,
+        });
+
+        CaptionFile.waitForCaptionCueGeneration(id).then(() => {
+          this.selectedLanguage = null;
+        });
+      },
+      getLongestDurationFileId() {
+        const files = this.getContentNodeFiles(this.nodeId);
+        const { id } = files.reduce((max, file) => (file.duration > max.duration ? file : max), {
+          duration: 0,
+        });
+        return id;
+      },
+    },
+    $trs: {
+      header: 'Add Captions for {fileName}',
+      generateBtn: 'Generate Captions',
+      captionGenerationHelpText:
+        'Select a language to automatically generate captions for this video',
+      generating: 'Generating caption for file: {fileName}',
+    },
+  };
+
+</script>
+
+<style lang="less" scoped>
+
+  .captions-tab-section {
+    height: 80vh;
+    vertical-align: text-top;
+  }
+
+  .title {
+    display: inline-flex;
+    align-items: center;
+    justify-content: flex-start;
+  }
+
+</style>

contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue

@@ -340,6 +340,7 @@
               // (especially marking nodes as (in)complete)
               vm.loadFiles({ contentnode__in: childrenNodesIds }),
               vm.loadAssessmentItems({ contentnode__in: childrenNodesIds }),
+              vm.loadCaptions({ contentnode__in: childrenNodesIds }),
             ];
           } else {
             // no need to load assessment items or files as topics have none
@@ -394,6 +395,7 @@
       ]),
       ...mapActions('file', ['loadFiles', 'updateFile']),
       ...mapActions('assessmentItem', ['loadAssessmentItems', 'updateAssessmentItems']),
+      ...mapActions('caption', ['loadCaptions']),
       ...mapMutations('contentNode', { enableValidation: 'ENABLE_VALIDATION_ON_NODES' }),
       closeModal() {
         this.promptUploading = false;

contentcuration/contentcuration/frontend/channelEdit/components/edit/EditView.vue

@@ -62,6 +62,16 @@
                 {{ relatedResourcesCount }}
               </VChip>
             </VTab>
+
+            <!-- Captions editing tab -->
+            <VTab
+              v-if="showCaptions"
+              ref="caption-tab"
+              :href="`#${tabs.CAPTIONS}`"
+              @click="trackTab('Captions')"
+            >
+            {{ $tr(tabs.CAPTIONS) }}
+            </VTab>
           </Tabs>
         </ToolBar>
         <VContainer fluid>
@@ -82,13 +92,19 @@
             <VTabItem :key="tabs.QUESTIONS" ref="questionwindow" :value="tabs.QUESTIONS" lazy>
               <AssessmentTab :nodeId="nodeIds[0]" />
             </VTabItem>
+
             <VTabItem
               :key="tabs.RELATED"
               :value="tabs.RELATED"
               lazy
             >
               <RelatedResourcesTab :nodeId="nodeIds[0]" />
             </VTabItem>
+
+            <VTabItem
+              :key="tabs.CAPTIONS" ref="captionswindow" :value="tabs.CAPTIONS" lazy>
+              <CaptionsTab :nodeId="nodeIds[0]" />
+            </VTabItem>
           </VTabsItems>
         </VContainer>
       </VFlex>
@@ -104,6 +120,7 @@
 
   import { TabNames } from '../../constants';
   import AssessmentTab from '../../components/AssessmentTab/AssessmentTab';
+  import CaptionsTab from '../../components/CaptionsTab/CaptionsTab'
   import RelatedResourcesTab from '../../components/RelatedResourcesTab/RelatedResourcesTab';
   import DetailsTabView from './DetailsTabView';
   import { ContentKindsNames } from 'shared/leUtils/ContentKinds';
@@ -113,11 +130,12 @@
   export default {
     name: 'EditView',
     components: {
-      DetailsTabView,
-      AssessmentTab,
-      RelatedResourcesTab,
-      Tabs,
-      ToolBar,
+        AssessmentTab,
+        CaptionsTab,
+        DetailsTabView,
+        RelatedResourcesTab,
+        Tabs,
+        ToolBar,
     },
     props: {
       nodeIds: {
@@ -143,6 +161,7 @@
         'getImmediateRelatedResourcesCount',
       ]),
       ...mapGetters('assessmentItem', ['getAssessmentItemsAreValid', 'getAssessmentItemsCount']),
+      ...mapGetters(['isAIFeatureEnabled']),
       firstNode() {
         return this.nodes.length ? this.nodes[0] : null;
       },
@@ -167,6 +186,14 @@
       showRelatedResourcesTab() {
         return this.oneSelected && this.firstNode && this.firstNode.kind !== 'topic';
       },
+      showCaptions() {
+        return (
+          this.oneSelected && 
+          this.firstNode && 
+          (this.firstNode.kind === 'video' || this.firstNode.kind === 'audio') &&
+          this.isAIFeatureEnabled
+        )
+      },
       countText() {
         const totals = reduce(
           this.nodes,
@@ -260,6 +287,8 @@
       questions: 'Questions',
       /** @see TabNames.RELATED */
       related: 'Related',
+      /** @see TabNames.CAPTIONS */
+      captions: 'Captions',
       /* eslint-enable kolibri/vue-no-unused-translations */
       noItemsToEditText: 'Please select resources or folders to edit',
       invalidFieldsToolTip: 'Some required information is missing',