From 8f27611911d9e6b4c6012e1af44b6c14cf6eaffb Mon Sep 17 00:00:00 2001 From: James Date: Mon, 12 Nov 2018 15:55:26 +1100 Subject: [PATCH 1/4] Preview seek/scrubbing thumbnails --- src/js/config/defaults.js | 7 + src/js/plugins/previewThumbnails.js | 499 ++++++++++++++++++++++++ src/js/plyr.js | 6 + src/sass/plugins/previewThumbnails.scss | 70 ++++ src/sass/plyr.scss | 1 + 5 files changed, 583 insertions(+) create mode 100644 src/js/plugins/previewThumbnails.js create mode 100644 src/sass/plugins/previewThumbnails.scss diff --git a/src/js/config/defaults.js b/src/js/config/defaults.js index 95de6951..70f12a80 100644 --- a/src/js/config/defaults.js +++ b/src/js/config/defaults.js @@ -342,6 +342,8 @@ const defaults = { loading: 'plyr--loading', hover: 'plyr--hover', tooltip: 'plyr__tooltip', + previewThumbnailContainer: 'plyr__preview-thumbnail-container', + previewScrubbingContainer: 'plyr__preview-scrubbing-container', cues: 'plyr__cues', hidden: 'plyr__sr-only', hideControls: 'plyr--hide-controls', @@ -395,6 +397,11 @@ const defaults = { enabled: false, publisherId: '', }, + + // Preview Thumbnails plugin + previewThumbnails: { + enabled: false, + } }; export default defaults; diff --git a/src/js/plugins/previewThumbnails.js b/src/js/plugins/previewThumbnails.js new file mode 100644 index 00000000..1993ae96 --- /dev/null +++ b/src/js/plugins/previewThumbnails.js @@ -0,0 +1,499 @@ +import { formatTime } from '../utils/time'; +import { on, once, toggleListener, triggerEvent } from '../utils/events'; +import { createElement, emptyElement, getAttributesFromSelector, getElement, getElements, hasClass, matches, removeElement, setAttributes, setFocus, toggleClass, toggleHidden } from '../utils/elements'; +import fetch from '../utils/fetch'; + +/** + * Preview thumbnails for seek hover and scrubbing + * Seeking: Hover over the seek bar (desktop only): shows a small preview container above the seek bar + * Scrubbing: Click and drag the seek bar (desktop and mobile): shows the preview image over the entire video, as if the video is scrubbing at very high speed + * + * Notes: + * - Thumbs are set via JS settings on Plyr init, not HTML5 'track' property. Using the track property would be a bit gross, because it doesn't support custom 'kinds'. kind=metadata might be used for something else, and we want to allow multiple thumbnails tracks. Tracks must have a unique combination of 'kind' and 'label'. We would have to do something like kind=metadata,label=thumbnails1 / kind=metadata,label=thumbnails2. Square peg, round hole + * - VTT info: the image URL is relative to the VTT, not the current document. But if the url starts with a slash, it will naturally be relative to the current domain. https://support.jwplayer.com/articles/how-to-add-preview-thumbnails + * - This implementation uses multiple separate img elements. Other implementations use background-image on one element. This would be nice and simple, but Firefox and Safari have flickering issues with replacing backgrounds of larger images. It seems that Youtube perhaps only avoids this because they don't have the option for high-res previews (even the fullscreen ones, when mousedown/seeking). Images appear over the top of each other, and previous ones are discarded once the new ones have been rendered + */ + +class PreviewThumbnails { + /** + * PreviewThumbnails constructor. + * @param {object} player + * @return {PreviewThumbnails} + */ + constructor(player) { + this.player = player; + this.thumbnailsDefs = []; + this.showingThumb = null; // Index of the currently displayed thumbnail + this.lastMousemoveEventTime = Date.now(); + this.mouseDown = false; + this.imageShowCounter = 0; + this.imageTryShowCounter = 0; + + if (this.enabled) { + this.load(); + } + } + + get enabled() { + return ( + this.player.isHTML5 && + this.player.isVideo && + this.player.config.previewThumbnails.enabled + ); + } + + load() { + this.getThumbnailsDefs() + .then(() => { + // Initiate DOM listeners so that our preview thumbnails can be used + this.listeners(); + + // Build HTML DOM elements + this.elements(); + + // Turn off the regular seek tooltip + this.player.config.tooltips.seek = false; + + // Check to see if thumb container size was specified manually in CSS + this.determineContainerAutoSizing(); + }); + } + + // Download VTT files and parse them + getThumbnailsDefs() { + return new Promise((resolve, reject) => { + if (!this.player.config.previewThumbnails.src) { + throw new Error('Missing previewThumbnails.src config attribute'); + } + + // previewThumbnails.src can be string or list. If string, convert into single-element list + const configSrc = this.player.config.previewThumbnails.src + const urls = typeof configSrc === 'string' ? [configSrc] : configSrc + const promises = []; + + // Loop through each src url. Download and process the VTT file, storing the resulting data in this.thumbnailsDefs + for (const url of urls) { + promises.push(this.getThumbnailDef(url)); + } + + Promise.all(promises) + .then(() => { + // Sort smallest to biggest (e.g., [120p, 480p, 1080p]) + this.thumbnailsDefs.sort((x, y) => x.height - y.height) + this.player.debug.log('Preview thumbnails: thumbnailsDefs: ' + JSON.stringify(this.thumbnailsDefs, null, 4)) + + resolve() + }); + }) + } + + // Process individual VTT file + getThumbnailDef (url) { + return new Promise((resolve, reject) => { + fetch(url) + .then(response => { + const thumbnailsDef = { + frames: this.parseVtt(response), + height: null, + urlPrefix: '', + }; + + // If the URLs don't start with '/', then we need to set their relative path to be the location of the VTT file + // If the URLs do start with '/', then they obviously don't need a prefix, so it will remain blank + if (!thumbnailsDef.frames[0].text.startsWith('/')) { + thumbnailsDef.urlPrefix = url.substring(0, url.lastIndexOf('/') + 1); + } + + // Download the first frame, so that we can determine/set the height of this thumbnailsDef + const tempImage = new Image(); + tempImage.src = thumbnailsDef.urlPrefix + thumbnailsDef.frames[0].text; + tempImage.onload = () => { + thumbnailsDef.height = tempImage.naturalHeight; + + this.thumbnailsDefs.push(thumbnailsDef); + + resolve(); + } + }) + }) + } + + /** + * Setup hooks for Plyr and window events + */ + listeners() { + // Mouse hover over seek bar + on.call( + this.player, + this.player.elements.progress, + 'mousemove', + event => { + // Wait until media has a duration + if (this.player.media.duration) { + // Calculate seek hover position as approx video seconds + const clientRect = this.player.elements.progress.getBoundingClientRect(); + const percentage = 100 / clientRect.width * (event.pageX - clientRect.left); + this.seekTime = this.player.media.duration * (percentage / 100); + if (this.seekTime < 0) this.seekTime = 0; // The mousemove fires for 10+px out to the left + if (this.seekTime > this.player.media.duration - 1) this.seekTime = this.player.media.duration - 1; // Took 1 second off the duration for safety, because different players can disagree on the real duration of a video + this.mousePosX = event.pageX; + + // Set time text inside image container + this.player.elements.display.previewThumbnailTimeText.innerText = formatTime(this.seekTime); + + // Download and show image + this.showImageAtCurrentTime(); + } + } + ); + + // Touch device seeking - performs same function as above + on.call( + this.player, + this.player.elements.progress, + 'touchmove', + event => { + // Wait until media has a duration + if (this.player.media.duration) { + // Calculate seek hover position as approx video seconds + this.seekTime = this.player.media.duration * (this.player.elements.inputs.seek.value / 100); + + // Download and show image + this.showImageAtCurrentTime(); + } + } + ); + + // Hide thumbnail preview - on mouse click, mouse leave, and video play/seek. All four are required, e.g., for buffering + on.call( + this.player, + this.player.elements.progress, + 'mouseleave click', + () => { + this.hideThumbContainer(); + } + ); + this.player.on('play', () => { + this.hideThumbContainer(); + }); + this.player.on('seeked', () => { + this.hideThumbContainer(); + }); + + // Show scrubbing preview + on.call( + this.player, + this.player.elements.progress, + 'mousedown touchstart', + () => { + this.mouseDown = true; + this.showScrubbingContainer(); + this.hideThumbContainer(); + } + ); + on.call( + this.player, + this.player.media, + 'timeupdate', + () => { + this.timeAtLastTimeupdate = this.player.media.currentTime; + } + ); + on.call( + this.player, + this.player.elements.progress, + 'mouseup touchend', + () => { + this.mouseDown = false; + + // Hide scrubbing preview. But wait until the video has successfully seeked before hiding the scrubbing preview + if (Math.ceil(this.timeAtLastTimeupdate) === Math.ceil(this.player.media.currentTime)) { + // The video was already seeked/loaded at the chosen time - hide immediately + this.hideScrubbingContainer(); + } else { + // The video hasn't seeked yet. Wait for that + once.call( + this.player, + this.player.media, + 'timeupdate', + () => { + // Re-check mousedown - we might have already started scrubbing again + if (!this.mouseDown) { + this.hideScrubbingContainer(); + } + } + ); + } + } + ); + } + + /** + * Create HTML elements for image containers + */ + elements() { + // Create HTML element: plyr__preview-thumbnail-container + const previewThumbnailContainer = createElement( + 'div', + { + class: this.player.config.classNames.previewThumbnailContainer, + }, + ); + + this.player.elements.progress.appendChild(previewThumbnailContainer); + this.player.elements.display.previewThumbnailContainer = previewThumbnailContainer; + + const timeText = createElement( + 'span', + {}, + '00:00', + ); + + this.player.elements.display.previewThumbnailContainer.appendChild(timeText); + this.player.elements.display.previewThumbnailTimeText = timeText; + + // Create HTML element: plyr__preview-scrubbing-container + const previewScrubbingContainer = createElement( + 'div', + { + class: this.player.config.classNames.previewScrubbingContainer, + }, + ); + + this.player.elements.wrapper.appendChild(previewScrubbingContainer); + this.player.elements.display.previewScrubbingContainer = previewScrubbingContainer; + } + + showImageAtCurrentTime () { + if (!this.mouseDown) { + this.showThumbContainer(); + } + + this.setThumbContainerSizeAndPos(); + + // Check when we last loaded an image - don't show more than one new one every 500ms + if (this.lastMousemoveEventTime < Date.now() - 150) { + this.lastMousemoveEventTime = Date.now(); + + // Find the first thumbnail that's after `time`. Note `this.seekTime+1` - we're actually looking 1 second ahead, because it's more likely then that the viewer will actually get to see the preview frame in the actual video. This hack should be removed if we ever choose to make it seek to the nearest thumb time + const thumbNum = this.thumbnailsDefs[0].frames.findIndex(frame => this.seekTime+1 >= frame.startTime && this.seekTime <= frame.endTime); + + // Only show if the thumbnail to show is different to last time + if (thumbNum !== this.showingThumb) { + this.showingThumb = thumbNum; + this.showImage(); + } + } else { + // Set a timeout so that we always fire this function once after the mouse stops moving. If not for this, the mouse preview would often be a bit stale + if (!this.mousemoveEventTimeout) { + this.mousemoveEventTimeout = setTimeout(() => { + // Don't follow through after the timeout if it's since been hidden + if (this.player.elements.display.previewThumbnailContainer.style.opacity === 1) { + this.showImageAtCurrentTime(); + this.mousemoveEventTimeout = null; + } + }, 200) + } + } + } + + // Show the image that's currently specified in this.showingThumb + showImage (qualityIndex = 0) { + this.imageTryShowCounter += 1; + const localImageTryShowCounter = this.imageTryShowCounter; + let thumbNum = this.showingThumb; + + if (thumbNum === this.thumbnailsDefs[qualityIndex].frames.length) { + // It can attempt to preview up to 5 seconds out past the end of the video. So we'll just show the last frame + thumbNum -= 1; + this.showingThumb = thumbNum; + } + + this.player.debug.log(`Preview thumbnails: showing thumbnum: ${thumbNum}: ${JSON.stringify(this.thumbnailsDefs[qualityIndex].frames[thumbNum])}`); + + const thumbFilename = this.thumbnailsDefs[qualityIndex].frames[thumbNum].text; + const urlPrefix = this.thumbnailsDefs[qualityIndex].urlPrefix; + const thumbURL = urlPrefix + thumbFilename; + + // We're building and adding a new image. In other implementations of similar functionality (Youtube), background image is instead used. But this causes issues with larger images in Firefox and Safari - switching between background images causes a flicker. Putting a new image over the top does not + const previewImage = new Image(); + previewImage.src = thumbURL; + previewImage.setAttribute('data-thumbnum', thumbNum); + + previewImage.onload = () => { + // Many images are loaded within milliseconds of each other. An earlier one might be the last one to finish loading. Make sure we don't show an images out of order + if (localImageTryShowCounter >= this.imageShowCounter) { + this.imageShowCounter = localImageTryShowCounter; + + this.currentContainer.appendChild(previewImage); + + // Now that this one is showing, start pre-loading a batch of nearby images. But only if this isn't a revisit + // this.preloadNearbyImages(thumbNum); + this.thumbnailsDefs[qualityIndex].frames[thumbNum].loaded = true + + this.removeOldImages(); + + // Look for a higher quality version of the same frame + if (qualityIndex < this.thumbnailsDefs.length - 1) { + // Only use the higher quality version if it's going to look any better - if the current thumb is of a lower pixel density than the thumbnail container + let previewContainerHeight = this.player.elements.display.previewThumbnailContainer.clientHeight; + if (this.mouseDown) previewContainerHeight = this.player.elements.display.previewScrubbingContainer.clientHeight; + // Adjust for HiDPI screen + if (window.devicePixelRatio) previewContainerHeight *= window.devicePixelRatio; + + if (previewImage.naturalHeight < previewContainerHeight) { + // Recurse this function - show a higher quality one, but only if the viewer is on this frame for a while + setTimeout(() => { + // Make sure the mouse hasn't already moved on and started hovering at another frame + if (this.showingThumb === thumbNum) { + this.showImage(qualityIndex + 1); + } + }, 150) + } + } + } + } + } + + // Not using this -- Preloading looked like maybe a good idea, but it seems to actually cause more trouble than it solves. Slow connections get really backed up. Fast connections don't really need it + // If we were to try using this again, we might need to look at not starting a second preload while another is still going? + preloadNearbyImages(thumbNum, amountToPreload=30) { + const actualShowingThumb = [...this.currentContainer.children].reverse()[0].getAttribute('data-thumbnum'); + if (actualShowingThumb && Number(actualShowingThumb) === this.showingThumb) { + let startNum = thumbNum - amountToPreload/2; + let endNum = thumbNum + amountToPreload/2; + if (startNum < 0) startNum = 0; + if (endNum > this.thumbnailsDefs[0].frames.length - 1) endNum = this.thumbnailsDefs[0].frames.length - 1; + + for (let i = startNum; i <= endNum; i++) { + if (!this.thumbnailsDefs[0].frames[i].loaded) { + this.player.debug.log('Thumbnail previews: preloading: ' + i); + + const thumbFilename = this.thumbnailsDefs[0].frames[i].text; + const urlPrefix = this.thumbnailsDefs[0].urlPrefix; + const thumbURL = urlPrefix + thumbFilename; + + // We're building and adding a new image. In other implementations of similar functionality (Youtube), background image is instead used. But this causes issues with larger images in Firefox and Safari - switching between background images causes a flicker. Putting a new image over the top does not + const previewImage = new Image(); + previewImage.src = thumbURL; + + // Set loaded attribute. This will prevent us from wasting CPU constantly trying to preload images that we already have loaded + this.thumbnailsDefs[0].frames[i].loaded = true; + } + } + } + } + + removeOldImages() { + // Get a list of all images, and reverse it - so that we can start from the end and delete all except for the most recent + const allImages = [...this.currentContainer.children].reverse(); + + // Start at the third image image - so we leave the last two images. Leaving only one might result in flickering if the newest one hasn't finished rendering yet + for (let i = 2; i < allImages.length; i++) { + if (allImages[i].tagName === 'IMG') { + this.currentContainer.removeChild(allImages[i]); + } + } + } + + get currentContainer() { + if (this.mouseDown) { + return this.player.elements.display.previewScrubbingContainer; + } else { + return this.player.elements.display.previewThumbnailContainer; + } + } + + showThumbContainer() { + this.player.elements.display.previewThumbnailContainer.style.opacity = 1; + } + hideThumbContainer() { + this.player.elements.display.previewThumbnailContainer.style.opacity = 0; + } + + showScrubbingContainer() { + this.player.elements.display.previewScrubbingContainer.style.opacity = 1; + } + hideScrubbingContainer() { + this.player.elements.display.previewScrubbingContainer.style.opacity = 0; + } + + determineContainerAutoSizing() { + if (this.player.elements.display.previewThumbnailContainer.clientHeight > 20) { + this.sizeSpecifiedInCSS = true; // This will prevent auto sizing in this.setThumbContainerSizeAndPos() + } + } + + // Set the size to be about a quarter of the size of video. Unless option dynamicSize === false, in which case it needs to be set in CSS + setThumbContainerSizeAndPos() { + // if (this.player.config.previewThumbnails.autoSize) { + if (!this.sizeSpecifiedInCSS) { + const videoAspectRatio = this.player.media.videoWidth / this.player.media.videoHeight; + const thumbHeight = this.player.elements.container.clientHeight / 4; + const thumbWidth = thumbHeight * videoAspectRatio; + this.player.elements.display.previewThumbnailContainer.style.height = `${thumbHeight}px`; + this.player.elements.display.previewThumbnailContainer.style.width = `${thumbWidth}px`; + } + + this.setThumbContainerPos(); + } + + setThumbContainerPos() { + const seekbarRect = this.player.elements.progress.getBoundingClientRect(); + const plyrRect = this.player.elements.container.getBoundingClientRect(); + const previewContainer = this.player.elements.display.previewThumbnailContainer; + + // Find the lowest and highest desired left-position, so we don't slide out the side of the video container + const minVal = (plyrRect.left - seekbarRect.left + 10); + const maxVal = (plyrRect.right - seekbarRect.left - (previewContainer.clientWidth) - 10); + + // Set preview container position to: mousepos, minus seekbar.left, minus half of previewContainer.clientWidth + let previewPos = this.mousePosX - seekbarRect.left - (previewContainer.clientWidth / 2); + if (previewPos < minVal) { + previewPos = minVal; + } + if (previewPos > maxVal) { + previewPos = maxVal; + } + previewContainer.style.left = previewPos + 'px'; + } + + // Arg: vttDataString example: "WEBVTT\n\n1\n00:00:05.000 --> 00:00:10.000\n1080p-00001.jpg" + parseVtt (vttDataString) { + const processedList = [] + const frames = vttDataString.split(/\r\n\r\n|\n\n|\r\r/) + + for (const frame of frames) { + const result = {} + + for (const line of frame.split(/\r\n|\n|\r/)) { + if (result.startTime == null) { + // The line with start and end times on it is the first line of interest + const matchTimes = line.match(/([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})( ?--> ?)([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})/) // Note that this currently ignores caption formatting directives that are optionally on the end of this line - fine for non-captions VTT + + if (matchTimes) { + result.startTime = Number(matchTimes[1]) * 60 * 60 + Number(matchTimes[2]) * 60 + Number(matchTimes[3]) + Number("0." + matchTimes[4]) + result.endTime = Number(matchTimes[6]) * 60 * 60 + Number(matchTimes[7]) * 60 + Number(matchTimes[8]) + Number("0." + matchTimes[9]) + } + } else { + // If we already have the startTime, then we're definitely up to the text line(s) + if (line.trim().length > 0) { + if (!result.text) { + result.text = line.trim() + } else { + result.text += '\n' + line.trim() + } + } + } + } + + if (result.text) { + processedList.push(result) + } + } + + return processedList + } +} + +export default PreviewThumbnails; diff --git a/src/js/plyr.js b/src/js/plyr.js index daebdadc..49fc7c5a 100644 --- a/src/js/plyr.js +++ b/src/js/plyr.js @@ -15,6 +15,7 @@ import Fullscreen from './fullscreen'; import Listeners from './listeners'; import media from './media'; import Ads from './plugins/ads'; +import PreviewThumbnails from './plugins/previewThumbnails'; import source from './source'; import Storage from './storage'; import support from './support'; @@ -306,6 +307,11 @@ class Plyr { // Seek time will be recorded (in listeners.js) so we can prevent hiding controls for a few seconds after seek this.lastSeekTime = 0; + + // Setup preview thumbnails if enabled + if (this.config.previewThumbnails.enabled) { + this.previewThumbnails = new PreviewThumbnails(this); + } } // --------------------------------------- diff --git a/src/sass/plugins/previewThumbnails.scss b/src/sass/plugins/previewThumbnails.scss new file mode 100644 index 00000000..92702e39 --- /dev/null +++ b/src/sass/plugins/previewThumbnails.scss @@ -0,0 +1,70 @@ +// -------------------------------------------------------------- +// Preview Thumbnails +// -------------------------------------------------------------- + +.plyr__preview-thumbnail-container { + background-color: rgba(0,0,0,0.5); + border: 1px solid rgba(0,0,0,0); // The background colour above applies to the area under the border - so appears to be a border of 0.5 opacity black + border-radius: 0px; + bottom: 100%; + box-shadow: $plyr-tooltip-shadow; + left: 50%; + line-height: 1.3; + margin-bottom: $plyr-tooltip-padding * 2; + opacity: 0; + pointer-events: none; + position: absolute; + transition: opacity 0.2s 0.1s ease; + white-space: nowrap; + z-index: 2; + + img { + position: absolute; + left: 0px; + right: 0px; + top: 0px; + bottom: 0px; + margin: auto; + height: 100%; + width: 100%; + border-radius: 0px; + } + + // Seek time text + span { + position: absolute; + bottom: 0px; + z-index: 3; + transform: translate(-50%,0); + background-color: rgba(0,0,0,0.55); + color: rgba(255,255,255,1); + padding: 4px 6px 3px 6px; + font-size: $plyr-font-size-small; + font-weight: $plyr-font-weight-regular; + } +} + +.plyr__preview-scrubbing-container { + position: absolute; + left: 0px; + right: 0px; + top: 0px; + bottom: 0px; + height: 100%; + width: 100%; + z-index: 1; + transition: opacity 0.3s 0.3s ease; + filter: blur(1px); + + img { + position: absolute; + left: 0px; + right: 0px; + top: 0px; + bottom: 0px; + margin: auto; + height: 100%; + width: 100%; + object-fit: contain; + } +} diff --git a/src/sass/plyr.scss b/src/sass/plyr.scss index 3d824f7d..468c534c 100644 --- a/src/sass/plyr.scss +++ b/src/sass/plyr.scss @@ -42,6 +42,7 @@ @import 'states/fullscreen'; @import 'plugins/ads'; +@import 'plugins/previewThumbnails'; @import 'utils/animation'; @import 'utils/hidden'; From e948bfd585a7ad0472c612eeda1f8fe8bbcaa3cb Mon Sep 17 00:00:00 2001 From: James Date: Thu, 13 Dec 2018 20:39:39 +1100 Subject: [PATCH 2/4] Preview seek: jpeg sprites + much more - Allow jpeg sprites - much snappier and more accurate - Fixed bug: right clicking the seek bar sticks on mousedown - Fixed bug: moving the mouse really quickly results in not updating the thumb - Fixed bug: if you mousedown but don't move mouse, it shows a stale image in the scrubbing container - Fixed bug: very first image shows as 0px - Fixed bug: stretches images when video isn't same aspect as player --- src/js/plugins/previewThumbnails.js | 378 +++++++++++++++--------- src/sass/plugins/previewThumbnails.scss | 18 +- 2 files changed, 245 insertions(+), 151 deletions(-) diff --git a/src/js/plugins/previewThumbnails.js b/src/js/plugins/previewThumbnails.js index 1993ae96..f1fe376d 100644 --- a/src/js/plugins/previewThumbnails.js +++ b/src/js/plugins/previewThumbnails.js @@ -23,11 +23,9 @@ class PreviewThumbnails { constructor(player) { this.player = player; this.thumbnailsDefs = []; - this.showingThumb = null; // Index of the currently displayed thumbnail this.lastMousemoveEventTime = Date.now(); this.mouseDown = false; - this.imageShowCounter = 0; - this.imageTryShowCounter = 0; + this.loadedImages = []; if (this.enabled) { this.load(); @@ -43,6 +41,9 @@ class PreviewThumbnails { } load() { + // Turn off the regular seek tooltip + this.player.config.tooltips.seek = false; + this.getThumbnailsDefs() .then(() => { // Initiate DOM listeners so that our preview thumbnails can be used @@ -51,9 +52,6 @@ class PreviewThumbnails { // Build HTML DOM elements this.elements(); - // Turn off the regular seek tooltip - this.player.config.tooltips.seek = false; - // Check to see if thumb container size was specified manually in CSS this.determineContainerAutoSizing(); }); @@ -88,7 +86,7 @@ class PreviewThumbnails { } // Process individual VTT file - getThumbnailDef (url) { + getThumbnailDef(url) { return new Promise((resolve, reject) => { fetch(url) .then(response => { @@ -109,6 +107,7 @@ class PreviewThumbnails { tempImage.src = thumbnailsDef.urlPrefix + thumbnailsDef.frames[0].text; tempImage.onload = () => { thumbnailsDef.height = tempImage.naturalHeight; + thumbnailsDef.width = tempImage.naturalWidth; this.thumbnailsDefs.push(thumbnailsDef); @@ -170,14 +169,14 @@ class PreviewThumbnails { this.player.elements.progress, 'mouseleave click', () => { - this.hideThumbContainer(); + this.hideThumbContainer(true); } ); this.player.on('play', () => { - this.hideThumbContainer(); + this.hideThumbContainer(true); }); this.player.on('seeked', () => { - this.hideThumbContainer(); + this.hideThumbContainer(false); }); // Show scrubbing preview @@ -185,10 +184,19 @@ class PreviewThumbnails { this.player, this.player.elements.progress, 'mousedown touchstart', - () => { - this.mouseDown = true; - this.showScrubbingContainer(); - this.hideThumbContainer(); + event => { + // Only act on left mouse button (0) + if (event.button === 0) { + this.mouseDown = true; + // Wait until media has a duration + if (this.player.media.duration) { + this.showScrubbingContainer(); + this.hideThumbContainer(false); + + // Download and show image + this.showImageAtCurrentTime(); + } + } } ); on.call( @@ -264,134 +272,145 @@ class PreviewThumbnails { this.player.elements.display.previewScrubbingContainer = previewScrubbingContainer; } - showImageAtCurrentTime () { - if (!this.mouseDown) { + showImageAtCurrentTime() { + if (this.mouseDown) { + this.setScrubbingContainerSize(); + } else { this.showThumbContainer(); + this.setThumbContainerSizeAndPos(); } - this.setThumbContainerSizeAndPos(); - + // // TODO: move this logic to // Check when we last loaded an image - don't show more than one new one every 500ms - if (this.lastMousemoveEventTime < Date.now() - 150) { - this.lastMousemoveEventTime = Date.now(); + // if (this.lastMousemoveEventTime < Date.now() - 150) { + // this.lastMousemoveEventTime = Date.now(); - // Find the first thumbnail that's after `time`. Note `this.seekTime+1` - we're actually looking 1 second ahead, because it's more likely then that the viewer will actually get to see the preview frame in the actual video. This hack should be removed if we ever choose to make it seek to the nearest thumb time - const thumbNum = this.thumbnailsDefs[0].frames.findIndex(frame => this.seekTime+1 >= frame.startTime && this.seekTime <= frame.endTime); + // Find the desired thumbnail index + const thumbNum = this.thumbnailsDefs[0].frames.findIndex(frame => this.seekTime >= frame.startTime && this.seekTime <= frame.endTime); + let qualityIndex = 0; - // Only show if the thumbnail to show is different to last time + // Check to see if we've already downloaded higher quality versions of this image + for (let i = 1; i < this.thumbnailsDefs.length; i++) { + if (this.loadedImages.includes(this.thumbnailsDefs[i].frames[thumbNum].text)) { + qualityIndex = i; + } + } + + // Only proceed if either thumbnum or thumbfilename has changed if (thumbNum !== this.showingThumb) { this.showingThumb = thumbNum; - this.showImage(); + this.loadImage(qualityIndex); } - } else { - // Set a timeout so that we always fire this function once after the mouse stops moving. If not for this, the mouse preview would often be a bit stale - if (!this.mousemoveEventTimeout) { - this.mousemoveEventTimeout = setTimeout(() => { - // Don't follow through after the timeout if it's since been hidden - if (this.player.elements.display.previewThumbnailContainer.style.opacity === 1) { - this.showImageAtCurrentTime(); - this.mousemoveEventTimeout = null; - } - }, 200) - } - } + + // } else { + // // Set a timeout so that we always fire this function once after the mouse stops moving. If not for this, the mouse preview would often be a bit stale + // if (this.mousemoveEventTimeout) { + // clearTimeout(this.mousemoveEventTimeout); + // } + // this.mousemoveEventTimeout = setTimeout(() => { + // // Don't follow through after the timeout if it's since been hidden + // if (this.player.elements.display.previewThumbnailContainer.style.opacity === '1') { + // console.log('show on timer') + // this.showImageAtCurrentTime(true); + // this.mousemoveEventTimeout = null; + // } + // }, 200) + // } } // Show the image that's currently specified in this.showingThumb - showImage (qualityIndex = 0) { - this.imageTryShowCounter += 1; - const localImageTryShowCounter = this.imageTryShowCounter; + loadImage(qualityIndex = 0) { let thumbNum = this.showingThumb; - if (thumbNum === this.thumbnailsDefs[qualityIndex].frames.length) { - // It can attempt to preview up to 5 seconds out past the end of the video. So we'll just show the last frame - thumbNum -= 1; - this.showingThumb = thumbNum; - } - this.player.debug.log(`Preview thumbnails: showing thumbnum: ${thumbNum}: ${JSON.stringify(this.thumbnailsDefs[qualityIndex].frames[thumbNum])}`); + const frame = this.thumbnailsDefs[qualityIndex].frames[thumbNum]; const thumbFilename = this.thumbnailsDefs[qualityIndex].frames[thumbNum].text; const urlPrefix = this.thumbnailsDefs[qualityIndex].urlPrefix; const thumbURL = urlPrefix + thumbFilename; - // We're building and adding a new image. In other implementations of similar functionality (Youtube), background image is instead used. But this causes issues with larger images in Firefox and Safari - switching between background images causes a flicker. Putting a new image over the top does not - const previewImage = new Image(); - previewImage.src = thumbURL; - previewImage.setAttribute('data-thumbnum', thumbNum); + // console.log('loading: ' + thumbFilename + '. num: ' + thumbNum + '. qual: ' + qualityIndex); - previewImage.onload = () => { - // Many images are loaded within milliseconds of each other. An earlier one might be the last one to finish loading. Make sure we don't show an images out of order - if (localImageTryShowCounter >= this.imageShowCounter) { - this.imageShowCounter = localImageTryShowCounter; + if (!this.currentImageElement || this.currentImageElement.getAttribute('data-thumbfilename') !== thumbFilename) { + // If we're already loading a previous image, remove its onload handler - we don't want it to load after this one + // Only do this if not using jpeg sprites. Without jpeg sprites we really want to show as many images as possible, as a best-effort + if (this.loadingImage && this.usingJpegSprites) this.loadingImage.onload = null; - this.currentContainer.appendChild(previewImage); + // We're building and adding a new image. In other implementations of similar functionality (Youtube), background image is instead used. But this causes issues with larger images in Firefox and Safari - switching between background images causes a flicker. Putting a new image over the top does not + const previewImage = new Image(); + previewImage.src = thumbURL; + previewImage.setAttribute('data-thumbnum', thumbNum); + previewImage.setAttribute('data-thumbfilename', thumbFilename); + // this.showingThumbFilename = this.thumbnailsDefs[qualityIndex].frames[thumbNum].text; + this.showingThumbFilename = thumbFilename; - // Now that this one is showing, start pre-loading a batch of nearby images. But only if this isn't a revisit - // this.preloadNearbyImages(thumbNum); - this.thumbnailsDefs[qualityIndex].frames[thumbNum].loaded = true + // For some reason, passing the named function directly causes it to execute immediately. So I've wrapped it in an anonymous function... + previewImage.onload = () => this.showImage(previewImage, frame, qualityIndex, thumbNum, thumbFilename, true); + this.loadingImage = previewImage; + this.removeOldImages(previewImage); + } else { + // Update the existing image + this.showImage(this.currentImageElement, frame, qualityIndex, thumbNum, thumbFilename, false); + this.currentImageElement.setAttribute('data-thumbnum', thumbNum); + this.removeOldImages(this.currentImageElement); + } + } - this.removeOldImages(); + showImage(previewImage, frame, qualityIndex, thumbNum, thumbFilename, newImage = true) { + // console.log('newimage: ' + newImage) + console.log('showing: ' + thumbFilename + '. num: ' + thumbNum + '. qual: ' + qualityIndex + '. newimg: ' + newImage); + this.setImageSizeAndOffset(previewImage, frame); - // Look for a higher quality version of the same frame - if (qualityIndex < this.thumbnailsDefs.length - 1) { - // Only use the higher quality version if it's going to look any better - if the current thumb is of a lower pixel density than the thumbnail container - let previewContainerHeight = this.player.elements.display.previewThumbnailContainer.clientHeight; - if (this.mouseDown) previewContainerHeight = this.player.elements.display.previewScrubbingContainer.clientHeight; - // Adjust for HiDPI screen - if (window.devicePixelRatio) previewContainerHeight *= window.devicePixelRatio; + if (newImage) { + this.currentContainer.appendChild(previewImage); + + this.currentImageElement = previewImage; + // this.removeOldImages(previewImage); - if (previewImage.naturalHeight < previewContainerHeight) { - // Recurse this function - show a higher quality one, but only if the viewer is on this frame for a while - setTimeout(() => { - // Make sure the mouse hasn't already moved on and started hovering at another frame - if (this.showingThumb === thumbNum) { - this.showImage(qualityIndex + 1); - } - }, 150) + if (!this.loadedImages.includes(thumbFilename)) this.loadedImages.push(thumbFilename); + } + + // Look for a higher quality version of the same frame + if (qualityIndex < this.thumbnailsDefs.length - 1) { + // Only use the higher quality version if it's going to look any better - if the current thumb is of a lower pixel density than the thumbnail container + // let previewContainerHeight = this.player.elements.display.previewThumbnailContainer.clientHeight; + // if (this.mouseDown) previewContainerHeight = this.player.elements.display.previewScrubbingContainer.clientHeight; + // // Adjust for HiDPI screen + // if (window.devicePixelRatio) previewContainerHeight *= window.devicePixelRatio; + + // if (previewImage.naturalHeight < previewContainerHeight) { + // Recurse this function - show a higher quality one, but only if the viewer is on this frame for a while + setTimeout(() => { + // Make sure the mouse hasn't already moved on and started hovering at another frame + // TODO: need to use filename instead of thumbnum, but need to use latest thumbnum instead of old thumbnum + // if (this.showingThumb === thumbNum) { + console.log(`${this.showingThumbFilename} ${thumbFilename}`) + if (this.showingThumbFilename === thumbFilename) { + // console.log('showing higher qual') + this.loadImage(qualityIndex + 1); } - } - } + }, 500) + // } } } - // Not using this -- Preloading looked like maybe a good idea, but it seems to actually cause more trouble than it solves. Slow connections get really backed up. Fast connections don't really need it - // If we were to try using this again, we might need to look at not starting a second preload while another is still going? - preloadNearbyImages(thumbNum, amountToPreload=30) { - const actualShowingThumb = [...this.currentContainer.children].reverse()[0].getAttribute('data-thumbnum'); - if (actualShowingThumb && Number(actualShowingThumb) === this.showingThumb) { - let startNum = thumbNum - amountToPreload/2; - let endNum = thumbNum + amountToPreload/2; - if (startNum < 0) startNum = 0; - if (endNum > this.thumbnailsDefs[0].frames.length - 1) endNum = this.thumbnailsDefs[0].frames.length - 1; - - for (let i = startNum; i <= endNum; i++) { - if (!this.thumbnailsDefs[0].frames[i].loaded) { - this.player.debug.log('Thumbnail previews: preloading: ' + i); - - const thumbFilename = this.thumbnailsDefs[0].frames[i].text; - const urlPrefix = this.thumbnailsDefs[0].urlPrefix; - const thumbURL = urlPrefix + thumbFilename; - - // We're building and adding a new image. In other implementations of similar functionality (Youtube), background image is instead used. But this causes issues with larger images in Firefox and Safari - switching between background images causes a flicker. Putting a new image over the top does not - const previewImage = new Image(); - previewImage.src = thumbURL; - - // Set loaded attribute. This will prevent us from wasting CPU constantly trying to preload images that we already have loaded - this.thumbnailsDefs[0].frames[i].loaded = true; - } - } - } - } - - removeOldImages() { + removeOldImages(currentImage) { // Get a list of all images, and reverse it - so that we can start from the end and delete all except for the most recent - const allImages = [...this.currentContainer.children].reverse(); + const allImages = [...this.currentContainer.children]; - // Start at the third image image - so we leave the last two images. Leaving only one might result in flickering if the newest one hasn't finished rendering yet - for (let i = 2; i < allImages.length; i++) { - if (allImages[i].tagName === 'IMG') { - this.currentContainer.removeChild(allImages[i]); + for (let image of allImages) { + if (image.tagName === 'IMG') { + const removeDelay = this.usingJpegSprites ? 200 : 1000; + + if (image.getAttribute('data-thumbnum') !== currentImage.getAttribute('data-thumbnum') && !image.getAttribute('data-deleting')) { + // Wait 200ms, as the new image can take some time to show on certain browsers (even though it was downloaded before showing). This will prevent flicker, and show some generosity towards slower clients + // First set attribute 'deleting' to prevent multi-handling of this on repeat firing of this function + image.setAttribute('data-deleting', 'true'); + setTimeout(() => { + this.currentContainer.removeChild(image); + // console.log('removing: ' + image.getAttribute('data-thumbfilename')); + }, removeDelay) + } } } } @@ -404,11 +423,58 @@ class PreviewThumbnails { } } + get usingJpegSprites() { + if (this.thumbnailsDefs[0].frames[0].w) { + return true; + } else { + return false; + } + } + + get thumbAspectRatio() { + if (this.usingJpegSprites) { + return this.thumbnailsDefs[0].frames[0].w / this.thumbnailsDefs[0].frames[0].h; + } else { + return this.thumbnailsDefs[0].width / this.thumbnailsDefs[0].height; + } + } + + get thumbContainerHeight() { + if (this.mouseDown) { + // return this.player.elements.container.clientHeight; + // return this.player.media.clientHeight; + return this.player.media.clientWidth / this.thumbAspectRatio; // Can't use media.clientHeight - html5 video goes big and does black bars above and below + } else { + // return this.player.elements.container.clientHeight / 4; + return this.player.media.clientWidth / this.thumbAspectRatio / 4; + } + } + + get currentImageElement() { + if (this.mouseDown) { + return this.currentScrubbingImageElement; + } else { + return this.currentThumbnailImageElement; + } + } + set currentImageElement(element) { + if (this.mouseDown) { + this.currentScrubbingImageElement = element; + } else { + this.currentThumbnailImageElement = element; + } + } + showThumbContainer() { this.player.elements.display.previewThumbnailContainer.style.opacity = 1; } - hideThumbContainer() { + hideThumbContainer(clearShowing = false) { this.player.elements.display.previewThumbnailContainer.style.opacity = 0; + + if (clearShowing) { + this.showingThumb = null; + this.showingThumbFilename = null; + } } showScrubbingContainer() { @@ -416,6 +482,8 @@ class PreviewThumbnails { } hideScrubbingContainer() { this.player.elements.display.previewScrubbingContainer.style.opacity = 0; + this.showingThumb = null; + this.showingThumbFilename = null; } determineContainerAutoSizing() { @@ -426,12 +494,9 @@ class PreviewThumbnails { // Set the size to be about a quarter of the size of video. Unless option dynamicSize === false, in which case it needs to be set in CSS setThumbContainerSizeAndPos() { - // if (this.player.config.previewThumbnails.autoSize) { if (!this.sizeSpecifiedInCSS) { - const videoAspectRatio = this.player.media.videoWidth / this.player.media.videoHeight; - const thumbHeight = this.player.elements.container.clientHeight / 4; - const thumbWidth = thumbHeight * videoAspectRatio; - this.player.elements.display.previewThumbnailContainer.style.height = `${thumbHeight}px`; + const thumbWidth = this.thumbContainerHeight * this.thumbAspectRatio; + this.player.elements.display.previewThumbnailContainer.style.height = `${this.thumbContainerHeight}px`; this.player.elements.display.previewThumbnailContainer.style.width = `${thumbWidth}px`; } @@ -458,41 +523,68 @@ class PreviewThumbnails { previewContainer.style.left = previewPos + 'px'; } + // Can't use 100% width, in case the video is a different aspect ratio to the video container + setScrubbingContainerSize() { + this.player.elements.display.previewScrubbingContainer.style.width = `${this.player.media.clientWidth}px`; + this.player.elements.display.previewScrubbingContainer.style.height = `${this.player.media.clientWidth/this.thumbAspectRatio}px`; // Can't use media.clientHeight - html5 video goes big and does black bars above and below + } + + // Jpeg sprites need to be offset to the correct location + setImageSizeAndOffset(previewImage, frame) { + if (this.usingJpegSprites) { + // Find difference between jpeg height and preview container height + const heightMulti = this.thumbContainerHeight / frame.h; + + previewImage.style.height = `${previewImage.naturalHeight * heightMulti}px`; + previewImage.style.width = `${previewImage.naturalWidth * heightMulti}px`; + previewImage.style.left = `-${Math.ceil(frame.x * heightMulti)}px`; + previewImage.style.top = `-${frame.y * heightMulti}px`; // todo: might need to round this one up too + } + } + // Arg: vttDataString example: "WEBVTT\n\n1\n00:00:05.000 --> 00:00:10.000\n1080p-00001.jpg" - parseVtt (vttDataString) { - const processedList = [] - const frames = vttDataString.split(/\r\n\r\n|\n\n|\r\r/) + parseVtt(vttDataString) { + const processedList = []; + const frames = vttDataString.split(/\r\n\r\n|\n\n|\r\r/); - for (const frame of frames) { - const result = {} + for (const frame of frames) { + const result = {}; - for (const line of frame.split(/\r\n|\n|\r/)) { - if (result.startTime == null) { - // The line with start and end times on it is the first line of interest - const matchTimes = line.match(/([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})( ?--> ?)([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})/) // Note that this currently ignores caption formatting directives that are optionally on the end of this line - fine for non-captions VTT + for (const line of frame.split(/\r\n|\n|\r/)) { + if (result.startTime == null) { + // The line with start and end times on it is the first line of interest + const matchTimes = line.match(/([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})( ?--> ?)([0-9]{2}):([0-9]{2}):([0-9]{2}).([0-9]{2,3})/) // Note that this currently ignores caption formatting directives that are optionally on the end of this line - fine for non-captions VTT - if (matchTimes) { - result.startTime = Number(matchTimes[1]) * 60 * 60 + Number(matchTimes[2]) * 60 + Number(matchTimes[3]) + Number("0." + matchTimes[4]) - result.endTime = Number(matchTimes[6]) * 60 * 60 + Number(matchTimes[7]) * 60 + Number(matchTimes[8]) + Number("0." + matchTimes[9]) + if (matchTimes) { + result.startTime = Number(matchTimes[1]) * 60 * 60 + Number(matchTimes[2]) * 60 + Number(matchTimes[3]) + Number("0." + matchTimes[4]) + result.endTime = Number(matchTimes[6]) * 60 * 60 + Number(matchTimes[7]) * 60 + Number(matchTimes[8]) + Number("0." + matchTimes[9]) + } + } else { + // If we already have the startTime, then we're definitely up to the text line(s) + if (line.trim().length > 0) { + if (!result.text) { + const lineSplit = line.trim().split('#xywh='); + result.text = lineSplit[0]; + + // If there's content in lineSplit[1], then we have jpeg sprites. If not, then it's just one frame per jpeg + if (lineSplit[1]) { + const xywh = lineSplit[1].split(','); + result.x = xywh[0]; + result.y = xywh[1]; + result.w = xywh[2]; + result.h = xywh[3]; + } + } + } + } } - } else { - // If we already have the startTime, then we're definitely up to the text line(s) - if (line.trim().length > 0) { - if (!result.text) { - result.text = line.trim() - } else { - result.text += '\n' + line.trim() - } + + if (result.text) { + processedList.push(result); } - } } - if (result.text) { - processedList.push(result) - } - } - - return processedList + return processedList; } } diff --git a/src/sass/plugins/previewThumbnails.scss b/src/sass/plugins/previewThumbnails.scss index 92702e39..044048eb 100644 --- a/src/sass/plugins/previewThumbnails.scss +++ b/src/sass/plugins/previewThumbnails.scss @@ -18,15 +18,16 @@ white-space: nowrap; z-index: 2; + overflow: hidden; + img { position: absolute; left: 0px; - right: 0px; top: 0px; - bottom: 0px; - margin: auto; - height: 100%; + height: 100%; // Non-jpeg-sprite images are 100%. Jpeg sprites will have their size applied by javascript width: 100%; + max-height: none; + max-width: none; border-radius: 0px; } @@ -52,19 +53,20 @@ bottom: 0px; height: 100%; width: 100%; + margin: auto; // Required when video is different dimensions to container (e.g., fullscreen) + overflow: hidden; z-index: 1; - transition: opacity 0.3s 0.3s ease; + transition: opacity 0.3s ease; filter: blur(1px); img { position: absolute; left: 0px; - right: 0px; top: 0px; - bottom: 0px; - margin: auto; height: 100%; width: 100%; + max-height: none; + max-width: none; object-fit: contain; } } From 279f0519053143c43f84d1b0e3511593d26533ae Mon Sep 17 00:00:00 2001 From: James Date: Fri, 14 Dec 2018 12:50:29 +1100 Subject: [PATCH 3/4] Preview seek: image preloading + tweaks/fixes - Preloads neighbouring images after showing current image - Re-fixed bug: if you mousedown but don't move mouse, it shows a stale image in the scrubbing container - Fixed bug: mobile device correctly detect touch --- src/js/plugins/previewThumbnails.js | 170 ++++++++++++++++------------ 1 file changed, 100 insertions(+), 70 deletions(-) diff --git a/src/js/plugins/previewThumbnails.js b/src/js/plugins/previewThumbnails.js index f1fe376d..71fdf0c7 100644 --- a/src/js/plugins/previewThumbnails.js +++ b/src/js/plugins/previewThumbnails.js @@ -185,13 +185,13 @@ class PreviewThumbnails { this.player.elements.progress, 'mousedown touchstart', event => { - // Only act on left mouse button (0) - if (event.button === 0) { + // Only act on left mouse button (0), or touch device (!event.button) + if (!event.button || event.button === 0) { this.mouseDown = true; // Wait until media has a duration if (this.player.media.duration) { this.showScrubbingContainer(); - this.hideThumbContainer(false); + this.hideThumbContainer(true); // Download and show image this.showImageAtCurrentTime(); @@ -280,57 +280,33 @@ class PreviewThumbnails { this.setThumbContainerSizeAndPos(); } - // // TODO: move this logic to - // Check when we last loaded an image - don't show more than one new one every 500ms - // if (this.lastMousemoveEventTime < Date.now() - 150) { - // this.lastMousemoveEventTime = Date.now(); + // Find the desired thumbnail index + const thumbNum = this.thumbnailsDefs[0].frames.findIndex(frame => this.seekTime >= frame.startTime && this.seekTime <= frame.endTime); + let qualityIndex = 0; - // Find the desired thumbnail index - const thumbNum = this.thumbnailsDefs[0].frames.findIndex(frame => this.seekTime >= frame.startTime && this.seekTime <= frame.endTime); - let qualityIndex = 0; - - // Check to see if we've already downloaded higher quality versions of this image - for (let i = 1; i < this.thumbnailsDefs.length; i++) { - if (this.loadedImages.includes(this.thumbnailsDefs[i].frames[thumbNum].text)) { - qualityIndex = i; - } + // Check to see if we've already downloaded higher quality versions of this image + for (let i = 1; i < this.thumbnailsDefs.length; i++) { + if (this.loadedImages.includes(this.thumbnailsDefs[i].frames[thumbNum].text)) { + qualityIndex = i; } + } - // Only proceed if either thumbnum or thumbfilename has changed - if (thumbNum !== this.showingThumb) { - this.showingThumb = thumbNum; - this.loadImage(qualityIndex); - } - - // } else { - // // Set a timeout so that we always fire this function once after the mouse stops moving. If not for this, the mouse preview would often be a bit stale - // if (this.mousemoveEventTimeout) { - // clearTimeout(this.mousemoveEventTimeout); - // } - // this.mousemoveEventTimeout = setTimeout(() => { - // // Don't follow through after the timeout if it's since been hidden - // if (this.player.elements.display.previewThumbnailContainer.style.opacity === '1') { - // console.log('show on timer') - // this.showImageAtCurrentTime(true); - // this.mousemoveEventTimeout = null; - // } - // }, 200) - // } + // Only proceed if either thumbnum or thumbfilename has changed + if (thumbNum !== this.showingThumb) { + this.showingThumb = thumbNum; + this.loadImage(qualityIndex); + } } // Show the image that's currently specified in this.showingThumb loadImage(qualityIndex = 0) { let thumbNum = this.showingThumb; - this.player.debug.log(`Preview thumbnails: showing thumbnum: ${thumbNum}: ${JSON.stringify(this.thumbnailsDefs[qualityIndex].frames[thumbNum])}`); - const frame = this.thumbnailsDefs[qualityIndex].frames[thumbNum]; const thumbFilename = this.thumbnailsDefs[qualityIndex].frames[thumbNum].text; const urlPrefix = this.thumbnailsDefs[qualityIndex].urlPrefix; const thumbURL = urlPrefix + thumbFilename; - // console.log('loading: ' + thumbFilename + '. num: ' + thumbNum + '. qual: ' + qualityIndex); - if (!this.currentImageElement || this.currentImageElement.getAttribute('data-thumbfilename') !== thumbFilename) { // If we're already loading a previous image, remove its onload handler - we don't want it to load after this one // Only do this if not using jpeg sprites. Without jpeg sprites we really want to show as many images as possible, as a best-effort @@ -341,7 +317,6 @@ class PreviewThumbnails { previewImage.src = thumbURL; previewImage.setAttribute('data-thumbnum', thumbNum); previewImage.setAttribute('data-thumbfilename', thumbFilename); - // this.showingThumbFilename = this.thumbnailsDefs[qualityIndex].frames[thumbNum].text; this.showingThumbFilename = thumbFilename; // For some reason, passing the named function directly causes it to execute immediately. So I've wrapped it in an anonymous function... @@ -357,41 +332,22 @@ class PreviewThumbnails { } showImage(previewImage, frame, qualityIndex, thumbNum, thumbFilename, newImage = true) { - // console.log('newimage: ' + newImage) - console.log('showing: ' + thumbFilename + '. num: ' + thumbNum + '. qual: ' + qualityIndex + '. newimg: ' + newImage); + this.player.debug.log('Showing thumb: ' + thumbFilename + '. num: ' + thumbNum + '. qual: ' + qualityIndex + '. newimg: ' + newImage); this.setImageSizeAndOffset(previewImage, frame); if (newImage) { this.currentContainer.appendChild(previewImage); - this.currentImageElement = previewImage; - // this.removeOldImages(previewImage); if (!this.loadedImages.includes(thumbFilename)) this.loadedImages.push(thumbFilename); } - // Look for a higher quality version of the same frame - if (qualityIndex < this.thumbnailsDefs.length - 1) { - // Only use the higher quality version if it's going to look any better - if the current thumb is of a lower pixel density than the thumbnail container - // let previewContainerHeight = this.player.elements.display.previewThumbnailContainer.clientHeight; - // if (this.mouseDown) previewContainerHeight = this.player.elements.display.previewScrubbingContainer.clientHeight; - // // Adjust for HiDPI screen - // if (window.devicePixelRatio) previewContainerHeight *= window.devicePixelRatio; - - // if (previewImage.naturalHeight < previewContainerHeight) { - // Recurse this function - show a higher quality one, but only if the viewer is on this frame for a while - setTimeout(() => { - // Make sure the mouse hasn't already moved on and started hovering at another frame - // TODO: need to use filename instead of thumbnum, but need to use latest thumbnum instead of old thumbnum - // if (this.showingThumb === thumbNum) { - console.log(`${this.showingThumbFilename} ${thumbFilename}`) - if (this.showingThumbFilename === thumbFilename) { - // console.log('showing higher qual') - this.loadImage(qualityIndex + 1); - } - }, 500) - // } - } + // Preload images before and after the current one + // Show higher quality of the same frame + // Each step here has a short time delay, and only continues if still hovering/seeking the same spot. This is to protect slow connections from overloading + this.preloadNearby(thumbNum, true) + .then(this.preloadNearby(thumbNum, false)) + .then(this.getHigherQuality(qualityIndex, previewImage, frame, thumbFilename)); } removeOldImages(currentImage) { @@ -400,21 +356,95 @@ class PreviewThumbnails { for (let image of allImages) { if (image.tagName === 'IMG') { - const removeDelay = this.usingJpegSprites ? 200 : 1000; + const removeDelay = this.usingJpegSprites ? 500 : 1000; if (image.getAttribute('data-thumbnum') !== currentImage.getAttribute('data-thumbnum') && !image.getAttribute('data-deleting')) { // Wait 200ms, as the new image can take some time to show on certain browsers (even though it was downloaded before showing). This will prevent flicker, and show some generosity towards slower clients // First set attribute 'deleting' to prevent multi-handling of this on repeat firing of this function image.setAttribute('data-deleting', 'true'); + const currentContainer = this.currentContainer; // This has to be set before the timeout - to prevent issues switching between hover and scrub + setTimeout(() => { - this.currentContainer.removeChild(image); - // console.log('removing: ' + image.getAttribute('data-thumbfilename')); + currentContainer.removeChild(image); + this.player.debug.log('Removing thumb: ' + image.getAttribute('data-thumbfilename')); }, removeDelay) } } } } + // Preload images before and after the current one. Only if the user is still hovering/seeking the same frame + // This will only preload the lowest quality + preloadNearby(thumbNum, forward = true) { + return new Promise((resolve, reject) => { + setTimeout(() => { + const oldThumbFilename = this.thumbnailsDefs[0].frames[thumbNum].text; + + if (this.showingThumbFilename === oldThumbFilename) { + // Find the nearest thumbs with different filenames. Sometimes it'll be the next index, but in the case of jpeg sprites, it might be 100+ away + let thumbnailsDefsCopy + if (forward) { + thumbnailsDefsCopy = this.thumbnailsDefs[0].frames.slice(thumbNum); + } else { + thumbnailsDefsCopy = this.thumbnailsDefs[0].frames.slice(0, thumbNum).reverse(); + } + + let foundOne = false; + + for (const frame of thumbnailsDefsCopy) { + const newThumbFilename = frame.text; + + if (newThumbFilename !== oldThumbFilename) { + // Found one with a different filename. Make sure it hasn't already been loaded on this page visit + if (!this.loadedImages.includes(newThumbFilename)) { + foundOne = true; + this.player.debug.log('Preloading thumb filename: ' + newThumbFilename); + + const urlPrefix = this.thumbnailsDefs[0].urlPrefix; + const thumbURL = urlPrefix + newThumbFilename; + + const previewImage = new Image(); + previewImage.src = thumbURL; + previewImage.onload = () => { + this.player.debug.log('Preloaded thumb filename: ' + newThumbFilename); + if (!this.loadedImages.includes(newThumbFilename)) this.loadedImages.push(newThumbFilename); + + // We don't resolve until the thumb is loaded + resolve() + }; + } + + break; + } + } + + // If there are none to preload then we want to resolve immediately + if (!foundOne) resolve(); + } + }, 300) + }) + } + + // If user has been hovering current image for half a second, look for a higher quality one + getHigherQuality(currentQualityIndex, previewImage, frame, thumbFilename) { + if (currentQualityIndex < this.thumbnailsDefs.length - 1) { + // Only use the higher quality version if it's going to look any better - if the current thumb is of a lower pixel density than the thumbnail container + let previewImageHeight = previewImage.naturalHeight; + if (this.usingJpegSprites) previewImageHeight = frame.h; + + if (previewImageHeight < this.thumbContainerHeight) { + // Recurse back to the loadImage function - show a higher quality one, but only if the viewer is on this frame for a while + setTimeout(() => { + // Make sure the mouse hasn't already moved on and started hovering at another image + if (this.showingThumbFilename === thumbFilename) { + this.player.debug.log('Showing higher quality thumb for: ' + thumbFilename) + this.loadImage(currentQualityIndex + 1); + } + }, 300) + } + } + } + get currentContainer() { if (this.mouseDown) { return this.player.elements.display.previewScrubbingContainer; From d97257a5a93707fbbdc150226c9fbee8feb8aedb Mon Sep 17 00:00:00 2001 From: James Date: Sat, 15 Dec 2018 11:32:50 +1100 Subject: [PATCH 4/4] Preview seek: Edge+IE11 fixes - Fixed bug: Edge seek errors: Replaced array spread with Array.from() - Fixed IE11 bug: seek time was offset to the left. Required an extra container div to facilitate this --- src/js/config/defaults.js | 7 +++++-- src/js/plugins/previewThumbnails.js | 21 ++++++++++++++++----- src/sass/plugins/previewThumbnails.scss | 19 ++++++++++++------- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/js/config/defaults.js b/src/js/config/defaults.js index 70f12a80..6ac21f23 100644 --- a/src/js/config/defaults.js +++ b/src/js/config/defaults.js @@ -342,8 +342,6 @@ const defaults = { loading: 'plyr--loading', hover: 'plyr--hover', tooltip: 'plyr__tooltip', - previewThumbnailContainer: 'plyr__preview-thumbnail-container', - previewScrubbingContainer: 'plyr__preview-scrubbing-container', cues: 'plyr__cues', hidden: 'plyr__sr-only', hideControls: 'plyr--hide-controls', @@ -376,6 +374,11 @@ const defaults = { active: 'plyr--airplay-active', }, tabFocus: 'plyr__tab-focus', + previewThumbnails: { + thumbnailContainer: 'plyr__preview-thumbnail-container', + scrubbingContainer: 'plyr__preview-scrubbing-container', + timeTextContainer: 'plyr__preview-time-text-container', + }, }, // Embed attributes diff --git a/src/js/plugins/previewThumbnails.js b/src/js/plugins/previewThumbnails.js index 71fdf0c7..7ae077e6 100644 --- a/src/js/plugins/previewThumbnails.js +++ b/src/js/plugins/previewThumbnails.js @@ -244,27 +244,37 @@ class PreviewThumbnails { const previewThumbnailContainer = createElement( 'div', { - class: this.player.config.classNames.previewThumbnailContainer, + class: this.player.config.classNames.previewThumbnails.thumbnailContainer, }, ); this.player.elements.progress.appendChild(previewThumbnailContainer); this.player.elements.display.previewThumbnailContainer = previewThumbnailContainer; + // Create HTML element, parent+span: time text (e.g., 01:32:00) + const timeTextContainer = createElement( + 'div', + { + class: this.player.config.classNames.previewThumbnails.timeTextContainer + }, + ); + + this.player.elements.display.previewThumbnailContainer.appendChild(timeTextContainer); + const timeText = createElement( 'span', {}, '00:00', ); - this.player.elements.display.previewThumbnailContainer.appendChild(timeText); + timeTextContainer.appendChild(timeText); this.player.elements.display.previewThumbnailTimeText = timeText; // Create HTML element: plyr__preview-scrubbing-container const previewScrubbingContainer = createElement( 'div', { - class: this.player.config.classNames.previewScrubbingContainer, + class: this.player.config.classNames.previewThumbnails.scrubbingContainer, }, ); @@ -350,9 +360,10 @@ class PreviewThumbnails { .then(this.getHigherQuality(qualityIndex, previewImage, frame, thumbFilename)); } + // Remove all preview images that aren't the designated current image removeOldImages(currentImage) { - // Get a list of all images, and reverse it - so that we can start from the end and delete all except for the most recent - const allImages = [...this.currentContainer.children]; + // Get a list of all images, convert it from a DOM list to an array + const allImages = Array.from(this.currentContainer.children); for (let image of allImages) { if (image.tagName === 'IMG') { diff --git a/src/sass/plugins/previewThumbnails.scss b/src/sass/plugins/previewThumbnails.scss index 044048eb..4de90667 100644 --- a/src/sass/plugins/previewThumbnails.scss +++ b/src/sass/plugins/previewThumbnails.scss @@ -32,16 +32,21 @@ } // Seek time text - span { + .plyr__preview-time-text-container { position: absolute; bottom: 0px; + left: 0px; + right: 0px; + margin-bottom: 2px; z-index: 3; - transform: translate(-50%,0); - background-color: rgba(0,0,0,0.55); - color: rgba(255,255,255,1); - padding: 4px 6px 3px 6px; - font-size: $plyr-font-size-small; - font-weight: $plyr-font-weight-regular; + + span { + background-color: rgba(0,0,0,0.55); + color: rgba(255,255,255,1); + padding: 4px 6px 3px 6px; + font-size: $plyr-font-size-small; + font-weight: $plyr-font-weight-regular; + } } }