Feat: Implemented custom IndexedDbStorage (#416)

* refactor: segment-storage.ts

* refactor: optimize segment storage deletion logic

* refactor: Implemented segments-storage interface

* refactor: update segment locking logic in HybridLoader

* refactor: hasSegment function

* refactor: Remove unused segment storage related code

* refactor: Update segment storage initialization

* refactor: P2P configuration

* refactor: Custom segment storage handling

* refactor: Remove async keyword from destroy method in segments-storage.interface.ts

* fix: lint error

* refactor: Update segment storage initialization and handling

* refactor: Segments storage clear logic

* refactor: segments-storage-interface

* refactor: Files structure

* refactor: Improve clear segments storage logic

* docs: Add ISegmentStorage docs

* refactor: Improve stream time window handling in SegmentsMemoryStorage

* refactor: segments-storage interface

* refactor: Update initialize segment storage logic

* refactor: Update SegmentsStorage interface

* refactor: Added validation of customSegmentStorage from config

* refactor: Swap func params in correct order

* refactor: Update segment storage classes and interfaces

* fix: imports

* refactor: Naming

* refactor: Improve segment storage event handling

* refactor: Optimize segment memory storage

- Improve segment storage event handling
- Update segment storage classes and interfaces
- Swap function parameters in correct order
- Set memory storage limit based on user agent
- Clear segments based on memory storage limit

* refactor: Optimize segment memory storage and update segment storage classes and interfaces

- Refactored the segment-memory-storage.ts file to optimize the memory storage of segments.
- Updated the segment storage classes and interfaces to improve performance and efficiency.

* refactor: Update segment memory storage limit configuration

- Change the `segmentsMemoryStorageLimit` configuration in the `Core` class to allow for an undefined value, instead of a specific number. This provides more flexibility in managing the memory storage limit for segments.

- Update the `CommonCoreConfig` type definition in the `types.ts` file to reflect the change in the `segmentsMemoryStorageLimit` property.

* refactor: Add segment categories in clear logic

This commit optimizes the segment memory storage by introducing segment storage categories. The new SegmentCategories type is added to classify segments into different categories such as obsolete, beyondHalfHttpWindowBehind, behindPlayback, and aheadHttpWindow. The segment removal logic is updated to use these categories for better organization and efficiency.

* refactor: Update segment memory storage limit description

* refactor: Simplify segment memory storage limit configuration

* refactor: Simplify segment memory storage limit configuration and optimize segment memory storage

* refactor: Improve clear logic and added getAvailableSpace func

* refactor: Simplify segment memory storage limit configuration and optimize segment memory storage

- Added a new function getAvailableMemoryPercent() to calculate the available memory percentage.
- Updated the generateQueue() function to pass the available memory percentage to QueueUtils.generateQueue().
- Modified the getUsedMemory() function in SegmentMemoryStorage to return the memory limit and memory used.
- Updated the getSegmentPlaybackStatuses() function in utils/stream.ts to calculate the time windows based on the available memory percentage.

* refactor: Disable random http downloads if memory storage is running out of memory

* refactor: Clear logic

* Revert "refactor: Clear logic"

This reverts commit 8a631e7.

* refactor: Improve segment memory storage and clear logic

* refactor: Improve segment memory storage

* refactor: Naming

* feat: Implemented custom indexedDbStorage

* feat: Add player component with IndexedDB in demo package

* feat: Add source code link to IndexedDB example

* refactor: Improve segment memory storage interface and getUsage() logic

* Refactor segment-memory-storage.ts: Swap parameters in getStoredSegmentIds()

* refactor: Swap parameters in getSegmentData()

* refactor: Update setSegmentChangeCallback parameter name

* refactor: Use updated storage interface

---------

Co-authored-by: Andriy Lysnevych <[email protected]>

packages/p2p-media-loader-demo/src/components/P2PVideoDemo.tsx

@@ -20,6 +20,7 @@ import { ShakaPlyr } from "./players/shaka/ShakaPlyr";
 import { HlsJsP2PEngine } from "p2p-media-loader-hlsjs";
 import { HlsjsVidstack } from "./players/hlsjs/HlsjsVidstack";
 import { PeerDetails } from "p2p-media-loader-core";
+import { HlsjsVidstackIndexedDB } from "./players/hlsjs/HlsjsVidstackIndexedDB";
 
 type DemoProps = {
   streamUrl?: string;
@@ -46,6 +47,7 @@ const playerComponents = {
   clappr_hls: HlsjsClapprPlayer,
   dplayer_hls: HlsjsDPlayer,
   hlsjs_hls: HlsjsPlayer,
+  vidstack_indexeddb_hls: HlsjsVidstackIndexedDB,
   shaka: Shaka,
   dplayer_shaka: ShakaDPlayer,
   clappr_shaka: ShakaClappr,

packages/p2p-media-loader-demo/src/components/players/hlsjs/Hlsjs.tsx

@@ -1,4 +1,4 @@
-import "./Hlsjs.css";
+import "./hlsjs.css";
 import { useEffect, useRef, useState } from "react";
 import { PlayerProps } from "../../../types";
 import { subscribeToUiEvents } from "../utils";

packages/p2p-media-loader-demo/src/components/players/hlsjs/HlsjsVidstackIndexedDB.tsx

@@ -0,0 +1,97 @@
+import "./vidstack_indexed_db.css";
+import "@vidstack/react/player/styles/default/theme.css";
+import "@vidstack/react/player/styles/default/layouts/video.css";
+import {
+  MediaPlayer,
+  MediaProvider,
+  isHLSProvider,
+  type MediaProviderAdapter,
+} from "@vidstack/react";
+import {
+  defaultLayoutIcons,
+  DefaultVideoLayout,
+} from "@vidstack/react/player/layouts/default";
+import { PlayerProps } from "../../../types";
+import { HlsJsP2PEngine, HlsWithP2PConfig } from "p2p-media-loader-hlsjs";
+import { subscribeToUiEvents } from "../utils";
+import { useCallback } from "react";
+import Hls from "hls.js";
+import { IndexedDbStorage } from "../../../custom-segment-storage-example/indexed-db-storage";
+
+export const HlsjsVidstackIndexedDB = ({
+  streamUrl,
+  announceTrackers,
+  onPeerConnect,
+  onPeerClose,
+  onChunkDownloaded,
+  onChunkUploaded,
+}: PlayerProps) => {
+  const onProviderChange = useCallback(
+    (provider: MediaProviderAdapter | null) => {
+      if (isHLSProvider(provider)) {
+        const HlsWithP2P = HlsJsP2PEngine.injectMixin(Hls);
+
+        provider.library = HlsWithP2P as unknown as typeof Hls;
+
+        const storageFactory = (_isLive: boolean) => new IndexedDbStorage();
+
+        const config: HlsWithP2PConfig<typeof Hls> = {
+          p2p: {
+            core: {
+              announceTrackers,
+              customSegmentStorageFactory: storageFactory,
+            },
+            onHlsJsCreated: (hls) => {
+              subscribeToUiEvents({
+                engine: hls.p2pEngine,
+                onPeerConnect,
+                onPeerClose,
+                onChunkDownloaded,
+                onChunkUploaded,
+              });
+            },
+          },
+        };
+
+        provider.config = config;
+      }
+    },
+    [
+      announceTrackers,
+      onChunkDownloaded,
+      onChunkUploaded,
+      onPeerConnect,
+      onPeerClose,
+    ],
+  );
+
+  return (
+    <div className="video-container">
+      <MediaPlayer
+        autoPlay
+        muted
+        onProviderChange={onProviderChange}
+        src={streamUrl}
+        playsInline
+      >
+        <MediaProvider />
+        <DefaultVideoLayout icons={defaultLayoutIcons} />
+      </MediaPlayer>
+
+      <div className="notice">
+        <p>
+          <strong>Note:</strong> Clearing of stored video segments is not
+          implemented in this example. To remove cached segments, please clear
+          your browser's IndexedDB manually.{" "}
+          <a
+            href="https://github.com/Novage/p2p-media-loader/tree/main/packages/p2p-media-loader-demo/src/custom-segment-storage-example"
+            target="_blank"
+            className="source-code-link"
+          >
+            View Source Code
+          </a>
+        </p>
+      </div>
+    </div>
+  );
+};

packages/p2p-media-loader-demo/src/components/players/hlsjs/vidstack_indexed_db.css

@@ -0,0 +1,22 @@
+.notice {
+  margin-top: 10px;
+  padding: 10px;
+  border: 1px solid #f0ad4e;
+  background-color: #fcf8e3;
+  border-radius: 4px;
+}
+
+.notice p {
+  margin: 0;
+  color: #8a6d3b;
+}
+
+.source-code-link {
+  color: #0275d8;
+  text-decoration: none;
+  font-weight: bold;
+}
+
+.source-code-link:hover {
+  text-decoration: underline;
+}

packages/p2p-media-loader-demo/src/constants.ts

@@ -8,6 +8,7 @@ export const PLAYERS = {
   plyr_hls: "Plyr",
   openPlayer_hls: "OpenPlayerJS",
   mediaElement_hls: "MediaElement",
+  vidstack_indexeddb_hls: "Vidstack IndexedDB example",
   shaka: "Shaka",
   dplayer_shaka: "DPlayer",
   clappr_shaka: "Clappr (DASH only)",

packages/p2p-media-loader-demo/src/custom-segment-storage-example/indexed-db-storage.ts

@@ -0,0 +1,238 @@
+import {
+  CommonCoreConfig,
+  SegmentStorage,
+  StreamConfig,
+  StreamType,
+} from "p2p-media-loader-core";
+import { IndexedDbWrapper } from "./indexed-db-wrapper";
+
+type SegmentDataItem = {
+  storageId: string;
+  data: ArrayBuffer;
+};
+
+type Playback = {
+  position: number;
+  rate: number;
+};
+
+type LastRequestedSegmentInfo = {
+  streamId: string;
+  segmentId: number;
+  startTime: number;
+  endTime: number;
+  swarmId: string;
+  streamType: StreamType;
+  isLiveStream: boolean;
+};
+
+type SegmentInfoItem = {
+  storageId: string;
+  dataLength: number;
+  streamId: string;
+  segmentId: number;
+  streamType: string;
+  startTime: number;
+  endTime: number;
+  swarmId: string;
+};
+
+function getStorageItemId(streamId: string, segmentId: number) {
+  return `${streamId}|${segmentId}`;
+}
+
+const INFO_ITEMS_STORE_NAME = "segmentInfo";
+const DATA_ITEMS_STORE_NAME = "segmentData";
+const DB_NAME = "p2p-media-loader";
+const DB_VERSION = 1;
+const BYTES_PER_MB = 1048576;
+
+export class IndexedDbStorage implements SegmentStorage {
+  private segmentsMemoryStorageLimit = 4000; // 4 GB
+  private currentMemoryStorageSize = 0; // current memory storage size in MB
+
+  private storageConfig?: CommonCoreConfig;
+  private mainStreamConfig?: StreamConfig;
+  private secondaryStreamConfig?: StreamConfig;
+  private cache = new Map<string, SegmentInfoItem>();
+
+  private currentPlayback?: Playback; // current playback position and rate
+  private lastRequestedSegment?: LastRequestedSegmentInfo; // details  about the last requested segment by the player
+  private db: IndexedDbWrapper;
+
+  private segmentChangeCallback?: (streamId: string) => void;
+
+  constructor() {
+    this.db = new IndexedDbWrapper(
+      DB_NAME,
+      DB_VERSION,
+      INFO_ITEMS_STORE_NAME,
+      DATA_ITEMS_STORE_NAME,
+    );
+  }
+
+  onPlaybackUpdated(position: number, rate: number) {
+    this.currentPlayback = { position, rate };
+  }
+
+  onSegmentRequested(
+    swarmId: string,
+    streamId: string,
+    segmentId: number,
+    startTime: number,
+    endTime: number,
+    streamType: StreamType,
+    isLiveStream: boolean,
+  ) {
+    this.lastRequestedSegment = {
+      streamId,
+      segmentId,
+      startTime,
+      endTime,
+      swarmId,
+      streamType,
+      isLiveStream,
+    };
+  }
+
+  async initialize(
+    storageConfig: CommonCoreConfig,
+    mainStreamConfig: StreamConfig,
+    secondaryStreamConfig: StreamConfig,
+  ) {
+    this.storageConfig = storageConfig;
+    this.mainStreamConfig = mainStreamConfig;
+    this.secondaryStreamConfig = secondaryStreamConfig;
+
+    try {
+      // await this.db.deleteDatabase();
+      await this.db.openDatabase();
+      await this.loadCacheMap();
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error("Failed to initialize custom segment storage:", error);
+      throw error;
+    }
+  }
+
+  async storeSegment(
+    swarmId: string,
+    streamId: string,
+    segmentId: number,
+    data: ArrayBuffer,
+    startTime: number,
+    endTime: number,
+    streamType: StreamType,
+    _isLiveStream: boolean,
+  ) {
+    const storageId = getStorageItemId(streamId, segmentId);
+    const segmentDataItem = {
+      storageId,
+      data,
+    };
+    const segmentInfoItem = {
+      storageId,
+      dataLength: data.byteLength,
+      streamId,
+      segmentId,
+      streamType,
+      startTime,
+      endTime,
+      swarmId,
+    };
+
+    try {
+      /*
+       * await this.clear();
+       * Implement your own logic to remove old segments and manage the memory storage size
+       */
+
+      await Promise.all([
+        this.db.put(DATA_ITEMS_STORE_NAME, segmentDataItem),
+        this.db.put(INFO_ITEMS_STORE_NAME, segmentInfoItem),
+      ]);
+
+      this.cache.set(storageId, segmentInfoItem);
+      this.increaseMemoryStorageSize(data.byteLength);
+
+      if (this.segmentChangeCallback) {
+        this.segmentChangeCallback(streamId);
+      }
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(`Failed to store segment ${segmentId}:`, error);
+      throw error;
+      // Optionally, implement retry logic or other error recovery mechanisms
+    }
+  }
+
+  async getSegmentData(_swarmId: string, streamId: string, segmentId: number) {
+    const segmentStorageId = getStorageItemId(streamId, segmentId);
+    try {
+      const result = await this.db.get<SegmentDataItem>(
+        DATA_ITEMS_STORE_NAME,
+        segmentStorageId,
+      );
+
+      return result?.data;
+    } catch (error) {
+      // eslint-disable-next-line no-console
+      console.error(
+        `Error retrieving segment data for ${segmentStorageId}:`,
+        error,
+      );
+      return undefined;
+    }
+  }
+
+  getUsage() {
+    /*
+     * Implement your own logic to calculate the memory used by the segments stored in memory.
+     */
+    return {
+      totalCapacity: this.segmentsMemoryStorageLimit,
+      usedCapacity: this.currentMemoryStorageSize,
+    };
+  }
+
+  hasSegment(_swarmId: string, streamId: string, segmentId: number) {
+    const storageId = getStorageItemId(streamId, segmentId);
+    return this.cache.has(storageId);
+  }
+
+  getStoredSegmentIds(streamId: string) {
+    const storedSegments: number[] = [];
+
+    for (const segment of this.cache.values()) {
+      if (segment.streamId === streamId) {
+        storedSegments.push(segment.segmentId);
+      }
+    }
+
+    return storedSegments;
+  }
+
+  destroy() {
+    this.db.closeDatabase();
+    this.cache.clear();
+  }
+
+  setSegmentChangeCallback(callback: (streamId: string) => void) {
+    this.segmentChangeCallback = callback;
+  }
+
+  private async loadCacheMap() {
+    const result = await this.db.getAll<SegmentInfoItem>(INFO_ITEMS_STORE_NAME);
+
+    result.forEach((item) => {
+      const storageId = getStorageItemId(item.streamId, item.segmentId);
+      this.cache.set(storageId, item);
+
+      this.increaseMemoryStorageSize(item.dataLength);
+    });
+  }
+
+  private increaseMemoryStorageSize(dataLength: number) {
+    this.currentMemoryStorageSize += dataLength / BYTES_PER_MB;
+  }
+}