Add enhance WebRTC handling with SEI metadata parsing

dist/RTCTransformWorker.worker.worker.js

@@ -0,0 +1,2 @@
+!function(){function t(t,r=""){return Array.from(t,(t=>("0"+(255&t).toString(16)).slice(-2))).join(r)}function r(r){const e=t(r);return[e.slice(0,8),e.slice(8,12),e.slice(12,16),e.slice(16,20),e.slice(20,24),e.slice(24,32)].join("-")}function e(r){const e=t(r);return parseInt(e,16)}function n(t,r){for(;r<t.byteLength-4;){if(0===t[r]&&0===t[r+1]&&(1===t[r+2]||0===t[r+2]&&1===t[r+3]))return r;r+=1}return-1}function a(){return new TransformStream({start(){},flush(){},async transform(a,s){(function(t){let r=0;const e=[];for(;r<t.byteLength-4;){const a=n(t,r);if(!(a>=r))break;{const s=n(t,a+(1===t[a+2]?3:4)+1);if(!(s>a)){e.push(t.subarray(a));break}e.push(t.subarray(a,s)),r=s}}return e})(new Uint8Array(a.data)).forEach((n=>{const a=1===n[2]?3:4;if(6==(31&n[a])){const s=function(t){const r=[];let e=0;const n=128===t[t.length-1]?t.length-1:t.length;for(;e<n;){let n=0;for(;255===t[e];)n+=255,e++;n+=t[e++];let a=0;for(;255===t[e];)a+=255,e++;a+=t[e++];const s=t.slice(e,e+a);return e+=a,r.push({type:n,size:a,payload:s}),{type:n,size:a,payload:s}}return r}(function(t){const r=[];for(let e=0;e<t.length;e++)e>2&&0===t[e-2]&&0===t[e-1]&&3===t[e]||r.push(t[e]);return new Uint8Array(r)}(n.subarray(a+1))),o={nalu:n,sei:s};"464d4c475241494e434f4c4f55524201"===t(s.payload.subarray(0,16))?postMessage({action:"sei",data:{...o,registered:!0,uuid:r(s.payload.subarray(0,16)),timecode:e(s.payload.subarray(16,24)),userdata:s.payload.subarray(24)}}):postMessage({action:"sei",data:{...o,registered:!1}})}})),s.enqueue(a)}})}function s({readable:t,writable:r},e){t.pipeThrough(e).pipeTo(r)}addEventListener("rtctransform",(t=>{s(t.transformer,a())})),addEventListener("message",(t=>{const{action:r}=t.data;"rtctransform"===r&&s(t.data,a())}))}();
+//# sourceMappingURL=RTCTransformWorker.worker.worker.js.map

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ovenplayer",
-  "version": "0.10.38",
+  "version": "0.10.39",
   "description": "OvenPlayer is Open-Source HTML5 Player. OvenPlayer supports WebRTC Signaling from OvenMediaEngine for Sub-Second Latency Streaming.",
   "main": "dist/ovenplayer.js",
   "scripts": {
@@ -39,7 +39,8 @@
     "webpack": "^5.51.1",
     "webpack-bundle-analyzer": "^3.0.3",
     "webpack-cli": "^4.8.0",
-    "webpack-merge": "^5.8.0"
+    "webpack-merge": "^5.8.0",
+    "worker-loader": "^3.0.8"
   },
   "dependencies": {
     "core-js": "^3.16.3",

src/js/api/Configurator.js

@@ -39,6 +39,9 @@ const Configurator = function (options, provider) {
             doubleTapToSeek: false,
             showZoomSettings: false,
             legacyUI: false,
+            parseStream: {
+                enabled: false
+            }
         };
         const serialize = function (val) {
             if (val === undefined) {

src/js/api/constants.js

@@ -68,6 +68,8 @@ export const CONTENT_RATE_CHANGE = "ratechange";
 export const CONTENT_VOLUME = "volumeChanged";
 export const CONTENT_MUTE = "mute";
 export const CONTENT_META = "metaChanged";
+export const CONTENT_META_DATA = "metaData";
+export const CONTENT_META_DATA_TYPE_SEI = "sei";
 export const CONTENT_SOURCE_CHANGED = "sourceChanged";
 export const CONTENT_LEVEL_CHANGED = "qualityLevelChanged";
 export const CONTENT_DURATION_CHANGED = "durationChanged";

src/js/api/provider/html5/providers/WebRTC.js

@@ -3,10 +3,11 @@
  */
 import Provider from "api/provider/html5/Provider";
 import WebRTCLoader from "api/provider/html5/providers/WebRTCLoader";
-import {isWebRTC} from "utils/validator";
-import {errorTrigger} from "api/provider/utils";
-import {PROVIDER_WEBRTC, ERROR, PLAYER_STATE, STATE_IDLE, STATE_LOADING} from "api/constants";
-import {ERRORS, PLAYER_WEBRTC_TIMEOUT} from "../../../constants";
+import { isWebRTC } from "utils/validator";
+import { errorTrigger } from "api/provider/utils";
+import { PROVIDER_WEBRTC, ERROR, PLAYER_STATE, STATE_IDLE, CONTENT_META_DATA, CONTENT_META_DATA_TYPE_SEI } from "api/constants";
+import { ERRORS, PLAYER_WEBRTC_TIMEOUT } from "../../../constants";
+import RTCTransformWorker from "../../../worker/RTCTransformWorker.worker";
 
 /**
  * @brief   webrtc provider extended core.
@@ -84,7 +85,7 @@ const WebRTC = function (element, playerConfig, adTagUrl) {
                 webrtcLoader = null;
             }
 
-            const loadCallback = function (stream) {
+            const loadCallback = function (e) {
 
                 if (element.srcObject) {
                     element.srcObject = null;
@@ -95,6 +96,8 @@ const WebRTC = function (element, playerConfig, adTagUrl) {
                     audioCtx = null;
                 }
 
+                const stream = e.streams[0];
+
                 element.srcObject = stream;
 
                 if (stream.getAudioTracks().length > 0) {
@@ -112,6 +115,37 @@ const WebRTC = function (element, playerConfig, adTagUrl) {
                     audioCtx.createMediaStreamSource(stream);
                 }
 
+                if (e.receiver.track.kind === 'video') {
+
+                    if (playerConfig.getConfig().parseStream.enabled) {
+                        const worker = new RTCTransformWorker();
+
+                        if ('RTCRtpScriptTransform' in window) {
+
+                            e.receiver.transform = new RTCRtpScriptTransform(worker);
+
+                        } else {
+
+                            const { readable, writable } = e.receiver.createEncodedStreams();
+
+                            worker.postMessage({
+                                action: 'rtctransform',
+                                readable,
+                                writable
+                            }, [readable, writable]);
+                        }
+
+                        worker.addEventListener('message', (event) => {
+
+                            if (event.data.action === 'sei') {
+                                that.trigger(CONTENT_META_DATA, {
+                                    ...event.data.data,
+                                    type: CONTENT_META_DATA_TYPE_SEI,
+                                });
+                            }
+                        });
+                    }
+                }
             };
 
             let internalErrorCallback = null;
@@ -203,13 +237,13 @@ const WebRTC = function (element, playerConfig, adTagUrl) {
         if (config.webrtcConfig) {
 
             if (typeof config.webrtcConfig.connectionTimeout === 'number'
-                    && config.webrtcConfig.connectionTimeout > 0) {
+                && config.webrtcConfig.connectionTimeout > 0) {
 
                 connectionTimeout = config.webrtcConfig.connectionTimeout;
             }
 
             if (typeof config.webrtcConfig.timeoutMaxRetry === 'number'
-                    && config.webrtcConfig.timeoutMaxRetry > 0) {
+                && config.webrtcConfig.timeoutMaxRetry > 0) {
 
                 timeoutMaxRetry = config.webrtcConfig.timeoutMaxRetry;
             }

src/js/api/provider/html5/providers/WebRTCLoader.js

@@ -359,7 +359,6 @@ const WebRTCLoader = function (provider,
 
                 peerConnection.setLocalDescription(desc).then(function () {
 
-
                 }).catch(function (error) {
 
                     let tempError = ERRORS.codes[PLAYER_WEBRTC_SET_LOCAL_DESC_ERROR];
@@ -457,8 +456,7 @@ const WebRTCLoader = function (provider,
                 extractLossPacketsOnNetworkStatus(mainPeerConnectionInfo);
             }
 
-            mainStream = e.streams[0];
-            loadCallback(e.streams[0]);
+            loadCallback(e);
 
             if (playerConfig.getConfig().webrtcConfig && playerConfig.getConfig().webrtcConfig.playoutDelayHint) {
 
@@ -481,9 +479,8 @@ const WebRTCLoader = function (provider,
 
                     OvenPlayerConsole.log("WebRTC playoutDelayHint", receiver, hint);
                 }
-
             }
-        };
+        }; // end of ontrack
     }
 
     function createClientPeerConnection(hostId, clientId) {

src/js/api/worker/RTCTransformWorker.worker.js

@@ -0,0 +1,206 @@
+/**
+ * Created by rock on 2025. 2
+ */
+
+const OVENMEDIAENGINE_SEI_METADATA_UUID = '464d4c475241494e434f4c4f55524201';
+
+function removeEmulationPreventionBytes(data) {
+  const rbsp = [];
+  for (let i = 0; i < data.length; i++) {
+
+    if (i > 2 && data[i - 2] === 0x00 && data[i - 1] === 0x00 && data[i] === 0x03) {
+      continue; // skip 0x03
+    }
+    rbsp.push(data[i]);
+  }
+  return new Uint8Array(rbsp);
+}
+
+function parseSEIPayload(rbsp) {
+
+  const messages = [];
+
+  let i = 0;
+  const rbspLength = rbsp[rbsp.length - 1] === 0x80 ? rbsp.length - 1 : rbsp.length;
+
+  while (i < rbspLength) {
+
+    let type = 0;
+    while (rbsp[i] === 0xFF) {
+      type += 255;
+      i++;
+    }
+    type += rbsp[i++];
+
+    let size = 0;
+    while (rbsp[i] === 0xFF) {
+      size += 255;
+      i++;
+    }
+    size += rbsp[i++];
+
+    const payload = rbsp.slice(i, i + size);
+    i += size;
+
+    messages.push({ type, size, payload });
+
+    return { type, size, payload };
+  }
+
+  return messages;
+}
+
+function toHexString(byteArray, delimiter = '') {
+  return Array.from(byteArray, byte => {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  }).join(delimiter);
+}
+
+function toHexArray(byteArray) {
+  return Array.from(byteArray, byte => {
+    return ('0' + (byte & 0xFF).toString(16)).slice(-2);
+  });
+}
+
+function toUUID(byteArray) {
+  const hexString = toHexString(byteArray);
+  return [
+    hexString.slice(0, 8),
+    hexString.slice(8, 12),
+    hexString.slice(12, 16),
+    hexString.slice(16, 20),
+    hexString.slice(20, 24),
+    hexString.slice(24, 32)
+  ].join('-');
+}
+
+function toTimestamp(byteArray) {
+  const hexString = toHexString(byteArray);
+  return parseInt(hexString, 16);
+}
+
+function toAsciiString(byteArray) {
+  return String.fromCharCode.apply(null, byteArray);
+}
+
+function findNalStartIndex(frameData, offset) {
+  while (offset < frameData.byteLength - 4) {
+    if ((frameData[offset] === 0x00 && frameData[offset + 1] === 0x00)
+      && (frameData[offset + 2] === 0x01 || (frameData[offset + 2] === 0x00 && frameData[offset + 3] === 0x01))) {
+      return offset;
+    } else {
+      offset += 1;
+    }
+  }
+  return -1;
+}
+
+function getNalus(frameData) {
+
+  let offset = 0;
+  const headerSize = 1;
+  const nalus = [];
+
+  while (offset < frameData.byteLength - 4) {
+
+    const startCodeIndex = findNalStartIndex(frameData, offset);
+
+    if (startCodeIndex >= offset) {
+
+      const startCodeLength = frameData[startCodeIndex + 2] === 0x01 ? 3 : 4;
+      const nextStartCodeIndex = findNalStartIndex(frameData, startCodeIndex + startCodeLength + headerSize);
+
+      if (nextStartCodeIndex > startCodeIndex) {
+
+        nalus.push(frameData.subarray(startCodeIndex, nextStartCodeIndex));
+        offset = nextStartCodeIndex;
+      } else {
+
+        nalus.push(frameData.subarray(startCodeIndex));
+        break;
+      }
+    } else {
+      break;
+    }
+  }
+  return nalus;
+}
+
+function createReceiverTransform() {
+  return new TransformStream({
+    start() { },
+    flush() { },
+    async transform(encodedFrame, controller) {
+
+      const nalus = getNalus(new Uint8Array(encodedFrame.data));
+
+      nalus.forEach((nalu) => {
+
+        const startCodeLength = nalu[2] === 0x01 ? 3 : 4;
+        const headerCodeLength = 1;
+        const nalHeader = nalu[startCodeLength];
+        const nalType = nalHeader & 0x1F;
+
+        // NAL Type SEI
+        if (nalType === 6) {
+
+          const rbsp = removeEmulationPreventionBytes(nalu.subarray(startCodeLength + headerCodeLength));
+
+          const parsedSei = parseSEIPayload(rbsp);
+
+          const eventData = {
+            nalu: nalu,
+            sei: parsedSei
+          };
+
+          const uuid = toHexString(parsedSei.payload.subarray(0, 16));
+
+          if (uuid === OVENMEDIAENGINE_SEI_METADATA_UUID) {
+
+            postMessage({
+              action: 'sei', data: {
+                ...eventData,
+                registered: true,
+                uuid: toUUID(parsedSei.payload.subarray(0, 16)),
+                timecode: toTimestamp(parsedSei.payload.subarray(16, 24)),
+                userdata: parsedSei.payload.subarray(24)
+              }
+            });
+          } else {
+
+            postMessage({
+              action: 'sei', data: {
+                ...eventData,
+                registered: false
+              }
+            });
+          }
+        }
+      });
+
+      controller.enqueue(encodedFrame);
+    }
+  })
+}
+
+function setupPipe({ readable, writable }, transform) {
+  readable
+    .pipeThrough(transform)
+    .pipeTo(writable)
+}
+
+addEventListener('rtctransform', (event) => {
+  setupPipe(event.transformer, createReceiverTransform());
+});
+
+addEventListener('message', (event) => {
+  const { action } = event.data;
+
+  switch (action) {
+    case 'rtctransform':
+      setupPipe(event.data, createReceiverTransform())
+      break;
+    default:
+      break;
+  }
+});

webpack.config.js

@@ -71,7 +71,11 @@ const config = {
             {
                 test: /\.(ttf|eot|svg|gif)(\?v=[0-9]\.[0-9]\.[0-9])?$/,
                 type: 'asset/inline'
-            }
+            },
+            {
+                test: /\.worker\.js$/,
+                use: { loader: "worker-loader" }
+            },
         ]
     },
     plugins: [