properly working with fmp4 and av1

README.md

@@ -0,0 +1,15 @@
+# hls-player.js
+
+Extremely basic hls player using MediaSource extensions.
+
+Meant to be a custom replacement for hls.js for the stream-server repo
+
+**Not a drop-in replacement nor intended for actual use**. Meant to just be used with the way stream-server handles the http serving.
+
+Currently only works with a single playlist file (not the manifest playlist)
+
+Only works with fragmented mp4, not mpegts. Requires h264 video and mp4a audio. Should be able to parse the codec profile automatically given the above.
+
+Use requires an html5 `<video>` tag. If attributes `autoplay muted` are set, can be used by adding this script: `<script>var x = new VideoLoader('video_tag_id')</script>`. Otherwise add extra code to handle playing, etc.
+
+Simplest export is `cat *.js > /path/to/output.js`. Minify, rename, do whatever extra bits you want.

hls-player.js

@@ -1,29 +1,45 @@
+// handles loading the playlist file
 class PlaylistLoader {
 	constructor() {
+		// to work with the stream-server impl
 		this.playlist_src = '/list/' + window.location.pathname.slice(6);
 		this.last_segment = null;
 		this.refresh_interval = null;
 		this.new_segments = [];
+		this.init_segment_uri = null;
 		this.fetch_playlist();
 	}
 
 	async fetch_playlist() {
 		const response = await fetch(this.playlist_src);
-		this.parse_playlist(await response.text());
+		// if returns a 404, means stream ended and playlist was deleted by cleanup
-		setTimeout(this.fetch_playlist.bind(this), this.refresh_interval * 1000);
+		// if success, parse the file and fetch playlist again after n ms
+		if (response.status == 200) {
+			this.parse_playlist(await response.text());
+			setTimeout(this.fetch_playlist.bind(this), this.refresh_interval * 1000);
+		}
 	}
 
+	// degraded m3u8 parsing
+	// impl only cares about segment uris, the target duration length
+	// assumes this is not the main manifest file so no switching
+	// assumes init fragment is init.mp4
 	parse_playlist(playlist_content) {
 		let lines = playlist_content.split('\n');
 		let segments = [];
-		let segment_block_flag = this.last_segment === null ? false : true;
+		let segment_block_flag = this.last_segment === null ? false : true; // for first passthrough
 		for (let i = 0; i < lines.length; i++) {
+			// sets segment fetch frequency
 			if (lines[i].startsWith("#")) {
 				if (lines[i].startsWith("EXT-X-TARGETDURATION", 1)) {
 					this.refresh_interval = parseFloat(lines[i].split(':')[1]);
+				} else if (lines[i].startsWith("EXT-X-MAP:URI", 1)) {
+					this.init_segment_uri = lines[i].split('=')[1].slice(1, -1);
 				}
 			} else {
 				if (segment_block_flag || lines[i] == '') {
+					// if the current segment line contains the last "acked" segment
+					// "ack" any following segments
 					if (lines[i] == this.last_segment) {
 						segment_block_flag = false;
 					}
@@ -32,16 +48,24 @@ class PlaylistLoader {
 				}
 			}
 		}
+		// add any segments acked to the segment fetch queue
 		this.new_segments = this.new_segments.concat(segments);
 		if (segments.length != 0) {
+			// update last acked segment to last newly acked segment if any
 			this.last_segment = segments.at(-1);
 		}
 	}
 }
 
+// handles the video fetching and source buffering
 class VideoLoader {
+	// entry point, should just be able to use this by adding this script:
+	// <script>let x = new VideoLoader('video_tag_id');</script>
 	constructor(vid_tag_id) {
+		// only relevant to uri construction
+		// jank to match the stream-server impl
 		this.stream_key = window.location.pathname.slice(6);
+		// init playlist loading as a check to see if playlist is even live
 		this.playlist_loader = new PlaylistLoader();
 		this.player = document.getElementById(vid_tag_id);
 		this.media_source = new MediaSource();
@@ -50,7 +74,12 @@ class VideoLoader {
 	}
 	
 	async prepare_buffer(_) {
-		let init_frag = await this.fetch_video('/vid/' + this.stream_key + '/init.mp4');
+		// fetch init.mp4 fragment and parse its mimetype
+		// codec info is needed for MediaSource compat
+		while (this.playlist_loader.init_segment_uri === null) {
+			await new Promise(r => setTimeout(r, 250));
+		}
+		let init_frag = await this.fetch_video(this.playlist_loader.init_segment_uri);
 		let mime = (new MP4Tree(init_frag)).get_mime();
 		if (!MediaSource.isTypeSupported(mime)) {
 			return;
@@ -58,14 +87,45 @@ class VideoLoader {
 		this.media_buffer = this.media_source.addSourceBuffer(mime);
 		this.media_buffer.mode = 'segments';
 		this.media_buffer.appendBuffer(init_frag);
-		this.fetch_new_segments();
+		await this.fetch_new_segments(); // next functions rely on some data being bufferred already
+		this.resync_video();
+		this.cleanup_old();
+	}
+
+	async cleanup_old() {
+		// remove old segments every 1 min
+		let timeout = 60000;
+		// if removal cannot happen since buffer is busy, try again in a bit
+		if (!this.media_buffer.updating && this.media_buffer.buffered.end(0) - 60 > this.media_buffer.buffered.start(0)) {
+			// remove segments that are older than 1min behind the current end of buffer
+			this.media_buffer.remove(0, this.media_buffer.buffered.end(0) - 60);
+		} else {
+			timeout = 250;
+		}
+		setTimeout(this.cleanup_old.bind(this), timeout);
+	}
+
+	async resync_video() {
+		if (!this.media_buffer.updating) {
+			let buffer_end = this.media_buffer.buffered.end(0);
+			// if video is over 15s behind, seek to 5s behind current end.
+			// not called on loop, only once on start
+			if (buffer_end - this.player.currentTime > 15) {
+				this.player.currentTime = buffer_end - 5;
+			}
+		}
 	}
 
 	async fetch_new_segments() {
+		// for each video in the new segments queue, fetch and append it
 		while (this.playlist_loader.new_segments.length > 0) {
-			let segment_uri = this.playlist_loader.new_segments.shift();
+			if (!this.media_buffer.updating) {
-			let segment_stream = await this.fetch_video(segment_uri);
+				let segment_uri = this.playlist_loader.new_segments.shift();
-			this.media_buffer.appendBuffer(segment_stream);
+				let segment_stream = await this.fetch_video(segment_uri);
+				this.media_buffer.appendBuffer(segment_stream);
+			}
+			// wait between each append try to let processing finish
+			await new Promise(r => setTimeout(r, 250));
 		}
 		setTimeout(this.fetch_new_segments.bind(this), this.playlist_loader.refresh_interval * 1000);
 	}

mp4-tree.js

@@ -1,3 +1,4 @@
+// add a getString to make reading box headers easier
 DataView.prototype.getString = function(offset, length) {
 	text = '';
 	let end = offset + length;
@@ -7,52 +8,137 @@ DataView.prototype.getString = function(offset, length) {
 	return text;
 }
 
-
+// mp4 tree traverser, as is only good for getting avc1 + mp4a codec info.
+// relevant tree parts:
+// root
+// |-moov
+//   |-trak
+//     |-mdia
+//       |-minf
+//         |-vmhd
+//         |-stbl
+//           |-stsd
+//             |-avc1
+//               |-avcC // has video codec profile
+//   |-trak
+//     |-mdia
+//       |-minf
+//         |-smhd
+//         |-stbl
+//           |-stsd
+//             |-mp4a
+//               |-esds // has audio codec profile
+//
+// headers are 4byte len + 4byte name string
+// most details taken from https://web.archive.org/web/20180219054429/http://l.web.umkc.edu/lizhu/teaching/2016sp.video-communication/ref/mp4.pdf
+// avc1 and mp4a specific header offsets from https://xhelmboyx.tripod.com/formats/mp4-layout.txt
 class MP4Tree {
 	constructor(data) {
 		this.data = new DataView(data);
 		this.codecs = [];
-		this.idx = 0;
+		this.idx = 0; // current byte index in data
 		this.parse_codecs();
 	}
 
 	parse_codecs() {
 		this.parse_into('moov', 8);
+		// loop over all traks (can handle pure audio, pure video, both, multiple)
+		// only doing h264 video since thats all I can check currently doing 
+		// transcoding and running obs together with cpu usage
 		while (this.idx < this.data.byteLength) {
-			if (!this.parse_until('trak')) {
+			if (!this.parse_until('trak')) { // if no more tracks in file
 				break;
 			}
-			const curr_track_end = this.idx + this.read_next_head().len;
+			let curr_track_end = this.idx + this.read_next_head().len;
 			this.idx += 8
 			this.parse_into('mdia', 8);
 			this.parse_into('minf', 8);
-			const track_type_head = this.read_next_head().name;
+			let track_type_head = this.read_next_head().name;
 			if (track_type_head == 'smhd' || track_type_head == 'vmhd') {
 				this.parse_into('stbl', 8);
 				this.parse_into('stsd', 16);
-				const media_sample = this.read_next_head().name;
+				let media_sample = this.read_next_head().name;
 				if (media_sample == 'avc1') {
 					this.parse_avc1();
+				} else if (media_sample == 'vp09') {
+					this.parse_vp09();
+				} else if (media_sample == 'av01') {
+					this.parse_av01();
 				} else if (media_sample == 'mp4a') {
 					this.parse_mp4a();
+				} else if (media_sample == 'Opus') {
+					this.codecs.push('opus');
+				} else {
+					this.codecs.push('unsup codec');
 				}
 			}
-			this.idx = curr_track_end;
+			this.idx = curr_track_end; // always skip to end of current processed track
 		}
 	}
 	
 	parse_avc1() {
-		this.parse_into('avc1', 86);
+		this.parse_into('avc1', 86); // avc1 has 78 bytes of frame metadata to skip
 		this.parse_into('avcC', 8);
-		const avc_profile = (this.data.getUint32(this.idx) & 0xffffff).toString(16);
+		// ex, 01 64 00 1f
+		// 01 is avc version (always 1, bit-and it off)
+		// rest is video codec profile, rep as hex
+		let avc_profile = (this.data.getUint32(this.idx) & 0xffffff).toString(16); 
 		this.codecs.push('avc1.' + avc_profile);
 	}
 
+	parse_vp09() {
+		this.parse_into('vp09', 86);
+		this.parse_into('vpcC', 12);
+		let profile = this.data.getUint8(this.idx).toString().padStart(2, "0");
+		let level = this.data.getUint8(this.idx + 1).toString();
+		let depth = (this.data.getUint8(this.idx + 2) >> 4).toString().padStart(2, "0");
+		this.codecs.push(["vp09", profile, level, depth].join('.'));
+	}
+
+	parse_av01() {
+		this.parse_into('av01', 86);
+		this.parse_into('av1C', 8);
+		let tmp = this.data.getUint8(this.idx + 1);
+		let profile = tmp >> 5;
+		let level = tmp & 0x1f;
+		let level_profile, bit_depth;
+		tmp = this.data.getUint8(this.idx + 2);
+		switch (tmp >> 7) {
+			case 0:
+				level_profile = "M";
+				break;
+			case 1:
+				level_profile = "H";
+				break;
+			default:
+				level_profile = "";
+		}
+		let depth_bits = (tmp >> 5) & 0x3;
+		switch (depth_bits) {
+			case 0:
+				bit_depth = "08";
+				break;
+			case 1:
+				bit_depth = "12";
+				break;
+			case 2:
+				bit_depth = "10";
+				break;
+			default:
+				bit_depth = "0";
+				break;
+		}
+		this.codecs.push(["av01", profile, level.toString().padStart(2, "0") + level_profile, bit_depth].join('.'))
+	}
+
 	parse_mp4a() {
-		this.parse_into('mp4a', 36);
+		this.parse_into('mp4a', 36); // 28 bytes of subboxes with unclear offsets
 		this.parse_into('esds', 12);
-		const oti = this.data.getUint8(this.idx + 13).toString(16);
+		// mp4a is as mp4a.40.2
-		const aot = (this.data.getUint8(this.idx + 31) >> 3).toString(10);
+		// oti has defined position, rep as hex
+		// aot as a subtype is defined as 5 bits of the fetched byte, rep as dec
+		let oti = this.data.getUint8(this.idx + 13).toString(16); 
+		let aot = (this.data.getUint8(this.idx + 31) >> 3).toString(10);
 		this.codecs.push(["mp4a", oti, aot].join('.'));
 	}
 
@@ -63,12 +149,16 @@ class MP4Tree {
 	get_codec_string() {
 		return this.codecs.join(', ');
 	}
-
+	
+	// find the given box, also descend into its child box depth
+	// offset is almost always 8, sometimes 12 or 16, defined in spec not file
 	parse_into(name, header_offset) {
 		this.parse_until(name);
 		this.idx += header_offset;
 	}
 
+	// find the box with the given name at the current box depth
+	// acts as though the current box extends to EOF.
 	parse_until(name) {
 		let curr_head = null;
 		while (this.idx + 8 < this.data.byteLength) {
@@ -82,6 +172,7 @@ class MP4Tree {
 		return false;
 	}
 	
+	// read the mp4 box header as if the idx is positioned there
 	read_next_head() {
 		return {"len": this.data.getUint32(this.idx), "name": this.data.getString(this.idx + 4, 4)};
 	}