diff --git a/src/actions/index.js b/src/actions/index.js index 0d67f00fc..892f7e672 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -5,7 +5,7 @@ import { athena as Athena, billing as Billing, devices as Devices, drives as Dri import MyCommaAuth from '@commaai/my-comma-auth'; import * as Types from './types'; -import { resetPlayback, selectLoop } from '../timeline/playback'; +import { selectLoop } from '../timeline/playback'; import {hasRoutesData } from '../timeline/segments'; import { getDeviceFromState, deviceVersionAtLeast, deviceIsOnline } from '../utils'; import { webrtcConnectionManager } from '../utils/webrtc'; @@ -163,7 +163,6 @@ export function urlForState(dongleId, log_id, start, end, prime) { function updateTimeline(state, dispatch, log_id, start, end, allowPathChange) { if (!state.loop || !state.loop.startTime || !state.loop.duration || state.loop.startTime < start || state.loop.startTime + state.loop.duration > end || state.loop.duration < end - start) { - dispatch(resetPlayback()); dispatch(selectLoop(start, end)); } diff --git a/src/actions/types.js b/src/actions/types.js index 0bbe668f1..2637c5261 100644 --- a/src/actions/types.js +++ b/src/actions/types.js @@ -7,7 +7,7 @@ export const ACTION_STARTUP_DATA = 'ACTION_STARTUP_DATA'; // global state management export const ACTION_SELECT_DEVICE = 'ACTION_SELECT_DEVICE'; export const ACTION_SELECT_TIME_FILTER = 'ACTION_SELECT_TIME_FILTER'; -export const ACTION_UPDATE_ROUTE_LIMIT = 'ACTION_UPDATE_ROUTE_LIMIT' +export const ACTION_UPDATE_ROUTE_LIMIT = 'ACTION_UPDATE_ROUTE_LIMIT'; export const ACTION_UPDATE_DEVICES = 'ACTION_UPDATE_DEVICES'; export const ACTION_UPDATE_DEVICE = 'ACTION_UPDATE_DEVICE'; export const ACTION_UPDATE_ROUTE = 'ACTION_UPDATE_ROUTE'; @@ -26,9 +26,11 @@ export const ACTION_PRIME_SUBSCRIBE_INFO = 'ACTION_PRIME_SUBSCRIBE_INFO'; export const ACTION_SEEK = 'action_seek'; export const ACTION_PAUSE = 'action_pause'; export const ACTION_PLAY = 'action_play'; +export const ACTION_PLAYBACK_SPEED = 'action_playback_speed'; export const ACTION_LOOP = 'action_loop'; export const ACTION_BUFFER_VIDEO = 'action_buffer_video'; export const ACTION_RESET = 'action_reset'; +export const ACTION_HAS_AUDIO = 'action_has_audio'; // segments export const ACTION_UPDATE_SEGMENT_RANGE = 'update_segment_range'; diff --git a/src/components/DriveMap/index.jsx b/src/components/DriveMap/index.jsx index 7abcc0acc..3507ef661 100644 --- a/src/components/DriveMap/index.jsx +++ b/src/components/DriveMap/index.jsx @@ -6,7 +6,6 @@ import raf from 'raf'; import ReactMapGL, { LinearInterpolator } from 'react-map-gl'; import { fetchDriveCoords } from '../../actions/cached'; -import { currentOffset } from '../../timeline'; import { DEFAULT_LOCATION, MAPBOX_STYLE, MAPBOX_TOKEN } from '../../utils/geocode'; const INTERACTION_TIMEOUT = 5000; @@ -46,7 +45,7 @@ class DriveMap extends Component { } componentDidUpdate(prevProps) { - const { dispatch, currentRoute, startTime } = this.props; + const { dispatch, currentRoute } = this.props; const prevRoute = prevProps.currentRoute?.fullname || null; const route = currentRoute?.fullname || null; @@ -57,10 +56,6 @@ class DriveMap extends Component { } } - if (prevProps.startTime && prevProps.startTime !== startTime) { - this.shouldFlyTo = true; - } - if (currentRoute && prevProps.currentRoute && currentRoute.driveCoords && prevProps.currentRoute.driveCoords !== currentRoute.driveCoords) { this.shouldFlyTo = false; @@ -99,7 +94,7 @@ class DriveMap extends Component { const markerSource = this.map && this.map.getMap().getSource('seekPoint'); if (markerSource) { if (this.props.currentRoute && this.props.currentRoute.driveCoords) { - const pos = this.posAtOffset(currentOffset()); + const pos = this.posAtOffset(this.props.offset); if (pos && pos.some((coordinate, index) => coordinate != this.lastMapPos[index])) { this.lastMapPos = pos; markerSource.setData({ @@ -310,7 +305,6 @@ class DriveMap extends Component { const stateToProps = Obstruction({ offset: 'offset', currentRoute: 'currentRoute', - startTime: 'startTime', }); export default connect(stateToProps)(DriveMap); diff --git a/src/components/DriveVideo/index.jsx b/src/components/DriveVideo/index.jsx index d4eb61e27..5e3cd0347 100644 --- a/src/components/DriveVideo/index.jsx +++ b/src/components/DriveVideo/index.jsx @@ -2,7 +2,6 @@ import React, { Component } from 'react'; import { connect } from 'react-redux'; import { CircularProgress, Typography } from '@material-ui/core'; -import debounce from 'debounce'; import Obstruction from 'obstruction'; import ReactPlayer from 'react-player/file'; @@ -10,9 +9,9 @@ import { video as Video } from '../../api'; import Colors from '../../colors'; import { ErrorOutline } from '../../icons'; -import { currentOffset } from '../../timeline'; -import { seek, bufferVideo } from '../../timeline/playback'; -import { isIos, isFirefox } from '../../utils/browser.js'; +import { bufferVideo, setPlaybackSpeed, resetPlayback, play, pause, seek } from '../../timeline/playback'; +import { setVideoPlayer, seekVideoPlayer, getVideoPlayerCurrentTime } from '../../timeline/videoPlayer'; +import { isIos } from '../../utils/browser.js'; const VideoOverlay = ({ loading, error }) => { let content; @@ -37,34 +36,18 @@ const VideoOverlay = ({ loading, error }) => { ); }; -const getVideoState = (videoPlayer) => { - const currentTime = videoPlayer.getCurrentTime(); - const { buffered } = videoPlayer.getInternalPlayer(); - - let bufferRemaining = -1; - for (let i = 0; i < buffered.length; i++) { - const end = buffered.end(i); - if (currentTime >= buffered.start(i) && currentTime <= end) { - bufferRemaining = end - currentTime; - break; - } - } - - return { - bufferRemaining, - hasLoaded: bufferRemaining > 0, - }; -}; - class DriveVideo extends Component { constructor(props) { super(props); this.onVideoBuffering = this.onVideoBuffering.bind(this); + this.onVideoBufferEnd = this.onVideoBufferEnd.bind(this); + this.onVideoPlay = this.onVideoPlay.bind(this); + this.onVideoPause = this.onVideoPause.bind(this); this.onHlsError = this.onHlsError.bind(this); this.onVideoError = this.onVideoError.bind(this); - this.onVideoResume = this.onVideoResume.bind(this); - this.syncVideo = debounce(this.syncVideo.bind(this), 200, true); + this.onVideoPlaybackRateChange = this.onVideoPlaybackRateChange.bind(this); + this.onTimeUpdate = this.onTimeUpdate.bind(this); this.firstSeek = true; this.videoPlayer = React.createRef(); @@ -76,69 +59,72 @@ class DriveVideo extends Component { } componentDidMount() { - const { playSpeed } = this.props; + const { dispatch } = this.props; + dispatch(resetPlayback()); + setVideoPlayer(this.videoPlayer.current); if (this.videoPlayer.current) { - this.videoPlayer.current.playbackRate = playSpeed || 1; + const internal = this.videoPlayer.current.getInternalPlayer(); + if (internal) { + internal.playbackRate = 1; + } } this.updateVideoSource({}); - this.syncVideo(); - this.videoSyncIntv = setInterval(this.syncVideo, 500); } componentDidUpdate(prevProps) { + const videoPlayer = this.videoPlayer.current; + setVideoPlayer(videoPlayer); this.updateVideoSource(prevProps); - this.syncVideo(); } componentWillUnmount() { - if (this.videoSyncIntv) { - clearTimeout(this.videoSyncIntv); - this.videoSyncIntv = null; - } + setVideoPlayer(null); } onVideoBuffering() { - const { dispatch, currentRoute } = this.props; - const videoPlayer = this.videoPlayer.current; - if (!videoPlayer || !currentRoute || !videoPlayer.getDuration()) { - dispatch(bufferVideo(true)); - } + const { dispatch } = this.props; + dispatch(bufferVideo(true)); + } - if (this.firstSeek) { - this.firstSeek = false; - videoPlayer.seekTo(this.currentVideoTime(), 'seconds'); - } + onVideoBufferEnd() { + const { dispatch } = this.props; + const { videoError } = this.state; + if (videoError) this.setState({ videoError: null }); + dispatch(bufferVideo(false)); + } + + onVideoPlay() { + const { dispatch } = this.props; + dispatch(play()); + dispatch(bufferVideo(false)); + } - const { hasLoaded } = getVideoState(videoPlayer); - const { readyState } = videoPlayer.getInternalPlayer(); - if (!hasLoaded || readyState < 2) { - dispatch(bufferVideo(true)); - } + onVideoPause() { + const { dispatch } = this.props; + dispatch(pause()); } - /** - * @param {Error} e - */ onHlsError(e) { const { dispatch } = this.props; dispatch(bufferVideo(true)); - if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { + if (!e.fatal) return; + else if (e.type === 'mediaError' && (e.details === 'bufferStalledError' || e.details === 'bufferNudgeOnStall')) { // buffer but no error return; - } - - if (e.type === 'networkError' && (e.response?.code === 404)) { + } else if (e.type === 'networkError' && (e.response?.code === 404)) { this.setState({ videoError: 'This video segment has not uploaded yet or has been deleted.' }); } else { - this.setState({ videoError: 'Unable to load video' }); + const message = + e.reason || + e.response?.text || + e.error?.message || + e.details || + 'Unknown playback error'; + this.setState({ videoError: message }); } } - /** - * @param {Error} e - * @param {any} [data] - */ onVideoError(e, data) { if (!e) { console.warn('Unknown video error', { e, data }); @@ -155,6 +141,14 @@ class DriveVideo extends Component { return; } + if (e.name === 'NotAllowedError') { + // autoplay was blocked (e.g. iOS after backgrounding/returning to the app) + const { dispatch } = this.props; + dispatch(bufferVideo(false)); + dispatch(pause()) + return; + } + if (e.target?.src?.startsWith(window.location.origin) && e.target.src.endsWith('undefined')) { // TODO: figure out why the src isn't set properly // Sometimes an error will be thrown because we try to play @@ -174,13 +168,31 @@ class DriveVideo extends Component { const videoError = e.response?.code === 404 ? 'This video segment has not uploaded yet or has been deleted.' - : (e.response?.text || 'Unable to load video'); + : (e.response?.text || e.message || 'Unable to load video'); this.setState({ videoError }); } - onVideoResume() { - const { videoError } = this.state; - if (videoError) this.setState({ videoError: null }); + onVideoPlaybackRateChange(rate) { + const { dispatch } = this.props; + dispatch(setPlaybackSpeed(rate)); + } + + onTimeUpdate(event) { + const { currentRoute, loop, dispatch } = this.props; + if (!currentRoute) { + return; + } + + const videoTime = getVideoPlayerCurrentTime(currentRoute); + if (videoTime >= loop.startTime + loop.duration) { + seekVideoPlayer(loop.startTime, currentRoute); + return; + } else if (videoTime < loop.startTime) { + seekVideoPlayer(loop.startTime, currentRoute); + return; + } + + dispatch(seek(videoTime)); } updateVideoSource(prevProps) { @@ -196,66 +208,11 @@ class DriveVideo extends Component { if (src === '' || !prevProps.currentRoute || prevProps.currentRoute?.fullname !== currentRoute.fullname) { src = Video.getQcameraStreamUrl(currentRoute.fullname, currentRoute.share_exp, currentRoute.share_sig); this.setState({ src, videoError: null }); - this.syncVideo(); - } - } - - syncVideo() { - const { dispatch, isBufferingVideo, isMuted } = this.props; - const videoPlayer = this.videoPlayer.current; - if (!videoPlayer || !videoPlayer.getInternalPlayer() || !videoPlayer.getDuration()) { - return; - } - - let { desiredPlaySpeed: newPlaybackRate } = this.props; - const desiredVideoTime = this.currentVideoTime(); - const curVideoTime = videoPlayer.getCurrentTime(); - const timeDiff = desiredVideoTime - curVideoTime; - - if (Math.abs(timeDiff) <= Math.max(0.1, 0.5 * newPlaybackRate)) { // newPlaybackRate = 0 when paused, set minimum 0.1 to prevent seeking when paused - if (!isIos()) { - newPlaybackRate = Math.max(0, newPlaybackRate + Math.round(timeDiff * 10) / 10); - } - } else if (desiredVideoTime === 0 && timeDiff < 0 && curVideoTime !== videoPlayer.getDuration()) { - // logs start earlier than video, so skip to video ts 0 - dispatch(seek(currentOffset() - (timeDiff * 1000))); - } else { - videoPlayer.seekTo(desiredVideoTime, 'seconds'); - } - // most browsers don't support more than 16x playback rate, firefox mutes audio above 8x causing audio to cut in and out with timeDiff rate shifts - newPlaybackRate = Math.max(0, Math.min((isFirefox() && !isMuted) ? 8 : 16, newPlaybackRate)); - - const internalPlayer = videoPlayer.getInternalPlayer(); - - const { hasLoaded } = getVideoState(videoPlayer); - if (isBufferingVideo && internalPlayer.readyState >= 4) { - dispatch(bufferVideo(false)); - } else if (isBufferingVideo || !hasLoaded || internalPlayer.readyState < 2) { - if (!isBufferingVideo) { - dispatch(bufferVideo(true)); - } - newPlaybackRate = 0; // in some circumstances, iOS won't update readyState unless temporarily paused - } - - if (videoPlayer.getInternalPlayer('hls')) { - if (!internalPlayer.paused && newPlaybackRate === 0) { - internalPlayer.pause(); - } else if (internalPlayer.playbackRate !== newPlaybackRate && newPlaybackRate !== 0) { - internalPlayer.playbackRate = newPlaybackRate; - } - if (internalPlayer.paused && newPlaybackRate !== 0) { - const playRes = internalPlayer.play(); - if (playRes) { - playRes.catch(() => console.debug('[DriveVideo] play interrupted by pause')); - } - } - } else { - // TODO: fix iOS bug where video doesn't stop buffering while paused - internalPlayer.playbackRate = newPlaybackRate; + this.firstSeek = true; } } - currentVideoTime(offset = currentOffset()) { + currentVideoTime(offset = this.props.offset) { const { currentRoute } = this.props; if (!currentRoute) { return 0; @@ -271,16 +228,24 @@ class DriveVideo extends Component { } render() { - const { desiredPlaySpeed, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted } = this.props; + const { isPlaying, isBufferingVideo, currentRoute, onAudioStatusChange, isMuted } = this.props; const { src, videoError } = this.state; const onPlayerReady = (player) => { + if (this.firstSeek) { + const video = player.getInternalPlayer(); + const startSeconds = this.currentVideoTime( + this.props.loop?.startTime || 0 + ); + video.currentTime = startSeconds; + this.firstSeek = false; + } + if (isIos()) { // ios does not support hls.js and on other browsers hls.js does not directly play the m3u8 so audioTracks are not visible const videoElement = player.getInternalPlayer(); - if (videoElement && videoElement.audioTracks && videoElement.audioTracks.length > 0) { - if (onAudioStatusChange) { - onAudioStatusChange(true); - } + const hasAudio = Boolean(videoElement && videoElement.audioTracks && videoElement.audioTracks.length > 0); + if (onAudioStatusChange) { + onAudioStatusChange(hasAudio); } } else { // on other platforms, inspect audio tracks before hls.js changes things const hlsPlayer = player.getInternalPlayer('hls'); @@ -295,29 +260,33 @@ class DriveVideo extends Component { }; return ( -
- - +
+
+ + +
); } @@ -325,12 +294,12 @@ class DriveVideo extends Component { const stateToProps = Obstruction({ dongleId: 'dongleId', - desiredPlaySpeed: 'desiredPlaySpeed', + isPlaying: 'isPlaying', offset: 'offset', - startTime: 'startTime', isBufferingVideo: 'isBufferingVideo', routes: 'routes', currentRoute: 'currentRoute', + loop: 'loop', }); export default connect(stateToProps)(DriveVideo); diff --git a/src/components/DriveView/Media.jsx b/src/components/DriveView/Media.jsx index 4ead69820..136712b54 100644 --- a/src/components/DriveView/Media.jsx +++ b/src/components/DriveView/Media.jsx @@ -17,7 +17,7 @@ import ResizeHandler from '../ResizeHandler'; import TimeDisplay from '../TimeDisplay'; import UploadQueue from '../Files/UploadQueue'; import SwitchLoading from '../utils/SwitchLoading'; -import { bufferVideo } from '../../timeline/playback'; +import { bufferVideo, setHasAudio } from '../../timeline/playback'; import Colors from '../../colors'; import { InfoOutline } from '../../icons'; import { deviceIsOnline, deviceOnCellular, getSegmentNumber } from '../../utils'; @@ -223,7 +223,6 @@ class Media extends Component { dcamUploadInfo: null, routePreserved: null, isMuted: true, - hasAudio: false, }; this.handleMuteToggle = this.handleMuteToggle.bind(this); @@ -251,7 +250,7 @@ class Media extends Component { } handleAudioStatusChange(hasAudio) { - this.setState({ hasAudio }); + this.props.dispatch(setHasAudio(hasAudio)); } componentDidMount() { @@ -298,12 +297,12 @@ class Media extends Component { } async copySegmentName() { - const { currentRoute } = this.props; + const { currentRoute, offset } = this.props; if (!currentRoute || !navigator.clipboard) { return; } - await navigator.clipboard.writeText(`${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute)}`); + await navigator.clipboard.writeText(`${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute, offset)}`); this.setState({ moreInfoMenu: null }); } @@ -339,7 +338,7 @@ class Media extends Component { } async uploadFile(type) { - const { dongleId, currentRoute } = this.props; + const { dongleId, currentRoute, offset } = this.props; if (!currentRoute) { return; } @@ -349,7 +348,7 @@ class Media extends Component { })); const routeNoDongleId = currentRoute.fullname.split('|')[1]; - const fileName = `${dongleId}|${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${type}`; + const fileName = `${dongleId}|${routeNoDongleId}--${getSegmentNumber(currentRoute, offset)}/${type}`; const uploading = {}; uploading[fileName] = { requested: true }; @@ -360,7 +359,7 @@ class Media extends Component { // request all possible file names for (const fn of FILE_NAMES[type]) { - const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute)}/${fn}`; + const path = `${routeNoDongleId}--${getSegmentNumber(currentRoute, offset)}/${fn}`; paths.push(path); url_promises.push(fetchUploadUrls(dongleId, [path]).then(urls => urls[0])); } @@ -522,7 +521,8 @@ class Media extends Component { render() { const { classes } = this.props; - const { inView, windowWidth, isMuted, hasAudio } = this.state; + const { inView, windowWidth, isMuted } = this.state; + const { hasAudio } = this.props; if (this.props.menusOnly) { // for test return this.renderMenus(true); @@ -530,26 +530,24 @@ class Media extends Component { const showMapAlways = windowWidth >= 1536; const mediaContainerStyle = showMapAlways ? { width: '60%' } : { width: '100%' }; - const mapContainerStyle = showMapAlways - ? { width: '40%', marginBottom: 62, marginTop: 46, paddingLeft: 24 } - : { width: '100%' }; return (
this.setState({ windowWidth: ww })} />
{this.renderMediaOptions(showMapAlways)} - {inView === MediaType.VIDEO && ( +
+ {/* always mounted -> keeps playing + driving the clock, even under the map */} - )} - {(inView === MediaType.MAP && !showMapAlways) && ( -
- -
- )} + {!showMapAlways && ( +
+ +
+ )} +
{(inView === MediaType.VIDEO && showMapAlways) && ( -
+
)} @@ -617,7 +615,7 @@ class Media extends Component { } renderMenus(alwaysOpen = false) { - const { currentRoute, device, classes, files, profile } = this.props; + const { currentRoute, device, classes, files, offset, profile } = this.props; const { downloadMenu, moreInfoMenu, uploadModal, windowWidth, dcamUploadInfo, routePreserved } = this.state; if (!device) { @@ -627,7 +625,7 @@ class Media extends Component { let fcam = {}; let ecam = {}; let dcam = {}; let rlog = {}; if (files && currentRoute) { - const seg = `${currentRoute.fullname}--${getSegmentNumber(currentRoute)}`; + const seg = `${currentRoute.fullname}--${getSegmentNumber(currentRoute, offset)}`; fcam = files[`${seg}/cameras`] || {}; ecam = files[`${seg}/ecameras`] || {}; dcam = files[`${seg}/dcameras`] || {}; @@ -758,7 +756,7 @@ class Media extends Component { onClick={ this.copySegmentName } style={{ fontSize: windowWidth > 400 ? '0.8rem' : '0.7rem' }} > -
{ currentRoute ? `${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute)}` : '---' }
+
{ currentRoute ? `${currentRoute.fullname.replace('|', '/')}/${getSegmentNumber(currentRoute, offset)}` : '---' }
{ typeof navigator.share !== 'undefined' @@ -895,10 +893,12 @@ const stateToProps = Obstruction({ device: 'device', routes: 'routes', currentRoute: 'currentRoute', + offset: 'offset', loop: 'loop', filter: 'filter', files: 'files', profile: 'profile', + hasAudio: 'hasAudio', isBufferingVideo: 'isBufferingVideo', }); diff --git a/src/components/TimeDisplay/index.jsx b/src/components/TimeDisplay/index.jsx index 1a340b1a4..7dd7d4a88 100644 --- a/src/components/TimeDisplay/index.jsx +++ b/src/components/TimeDisplay/index.jsx @@ -12,8 +12,7 @@ import VolumeOff from '@material-ui/icons/VolumeOff'; import { Tooltip } from '@material-ui/core'; import { DownArrow, Forward10, Pause, PlayArrow, Replay10, UpArrow } from '../../icons'; -import { currentOffset } from '../../timeline'; -import { seek, play, pause } from '../../timeline/playback'; +import { seekVideoPlayer, playVideo, pauseVideo, setVideoPlaybackRate, isVideoPaused } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; import { isIos } from '../../utils/browser.js'; @@ -101,16 +100,6 @@ const styles = (theme) => ({ }); class TimeDisplay extends Component { - static getDerivedStateFromProps(props, state) { - if (props.desiredPlaySpeed !== 0 && props.desiredPlaySpeed !== state.desiredPlaySpeed) { - return { - ...state, - desiredPlaySpeed: props.desiredPlaySpeed, - }; - } - return state; - } - constructor(props) { super(props); @@ -124,7 +113,6 @@ class TimeDisplay extends Component { this.jumpForward = this.jumpForward.bind(this); this.state = { - desiredPlaySpeed: 1, displayTime: this.getDisplayTime(), }; } @@ -139,14 +127,13 @@ class TimeDisplay extends Component { } getDisplayTime() { - const offset = currentOffset(); - const { currentRoute } = this.props; + const { currentRoute, offset } = this.props; const now = new Date(offset + currentRoute.start_time_utc_millis); if (Number.isNaN(now.getTime())) { return '...'; } let dateString = dayjs(now).format('HH:mm:ss'); - const seg = getSegmentNumber(currentRoute); + const seg = getSegmentNumber(currentRoute, offset); if (seg !== null) { dateString = `${dateString} \u2013 ${seg}`; } @@ -155,11 +142,15 @@ class TimeDisplay extends Component { } jumpBack(amount) { - this.props.dispatch(seek(currentOffset() - amount)); + const { currentRoute } = this.props; + const offset = this.props.offset - amount; + seekVideoPlayer(offset, currentRoute); } jumpForward(amount) { - this.props.dispatch(seek(currentOffset() + amount)); + const { currentRoute } = this.props; + const offset = this.props.offset + amount; + seekVideoPlayer(offset, currentRoute); } updateTime() { @@ -176,18 +167,18 @@ class TimeDisplay extends Component { } decreaseSpeed() { - const { dispatch } = this.props; - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); } curIndex = Math.max(0, curIndex - 1); - dispatch(play(timerSteps[curIndex])); + const newSpeed = timerSteps[curIndex]; + setVideoPlaybackRate(newSpeed); } canDecreaseSpeed() { - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); @@ -196,18 +187,18 @@ class TimeDisplay extends Component { } increaseSpeed() { - const { dispatch } = this.props; - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); } curIndex = Math.min(timerSteps.length - 1, curIndex + 1); - dispatch(play(timerSteps[curIndex])); + const newSpeed = timerSteps[curIndex]; + setVideoPlaybackRate(newSpeed); } canIncreaseSpeed() { - const { desiredPlaySpeed } = this.state; + const { desiredPlaySpeed } = this.props; let curIndex = timerSteps.indexOf(desiredPlaySpeed); if (curIndex === -1) { curIndex = timerSteps.indexOf(1); @@ -216,19 +207,16 @@ class TimeDisplay extends Component { } togglePause() { - const { desiredPlaySpeed, dispatch } = this.props; - if (desiredPlaySpeed === 0) { - // eslint-disable-next-line react/destructuring-assignment - dispatch(play(this.state.desiredPlaySpeed)); + if (isVideoPaused()) { + playVideo(); } else { - dispatch(pause()); + pauseVideo(); } } render() { - const { classes, zoom, desiredPlaySpeed: videoPlaySpeed, isThin, onMuteToggle, isMuted, hasAudio } = this.props; - const { displayTime, desiredPlaySpeed } = this.state; - const isPaused = videoPlaySpeed === 0; + const { classes, zoom, isThin, onMuteToggle, isMuted, hasAudio, desiredPlaySpeed, isPlaying } = this.props; + const { displayTime } = this.state; const isExpandedCls = zoom ? 'isExpanded' : ''; const isThinCls = isThin ? 'isThin' : ''; return ( @@ -302,9 +290,9 @@ class TimeDisplay extends Component {
- {isPaused + {!isPlaying ? () : ()} @@ -316,8 +304,10 @@ class TimeDisplay extends Component { const stateToProps = Obstruction({ currentRoute: 'currentRoute', + offset: 'offset', zoom: 'zoom', - desiredPlaySpeed: 'desiredPlaySpeed' + desiredPlaySpeed: 'desiredPlaySpeed', + isPlaying: 'isPlaying', }); export default connect(stateToProps)(withStyles(styles)(TimeDisplay)); diff --git a/src/components/Timeline/index.jsx b/src/components/Timeline/index.jsx index a3b7c66d4..fdee13d69 100644 --- a/src/components/Timeline/index.jsx +++ b/src/components/Timeline/index.jsx @@ -15,9 +15,9 @@ import Thumbnails from './thumbnails'; import theme from '../../theme'; import { pushTimelineRange } from '../../actions'; import Colors from '../../colors'; -import { currentOffset } from '../../timeline'; -import { seek } from '../../timeline/playback'; +import { getVideoPlayerCurrentTime, seekVideoPlayer } from '../../timeline/videoPlayer'; import { getSegmentNumber } from '../../utils'; +import { isIos } from '../../utils/browser.js'; const styles = () => ({ base: { @@ -168,6 +168,9 @@ class Timeline extends Component { this.dragBar = React.createRef(); this.hoverBead = React.createRef(); + this.currentOffset = null; + this.lastOffset = null; + const { zoomOverride, zoom } = this.props; this.state = { dragging: null, @@ -182,7 +185,7 @@ class Timeline extends Component { componentDidMount() { this.mounted = true; - raf(this.getOffset); + this.rafId = raf(this.getOffset); this.componentDidUpdate({}); } @@ -195,13 +198,18 @@ class Timeline extends Component { componentWillUnmount() { this.mounted = false; + if (this.rafId) { + raf.cancel(this.rafId); + this.rafId = null; + } } handleClick(ev) { const { dragging } = this.state; if (!dragging || Math.abs(dragging[1] - dragging[0]) <= 3) { const percent = percentFromPointerEvent(ev); - this.props.dispatch(seek(this.percentToOffset(percent))); + const offset = this.percentToOffset(percent); + seekVideoPlayer(offset, this.props.route); } } @@ -233,7 +241,7 @@ class Timeline extends Component { } handlePointerUp(ev) { - const { route } = this.props; + const { offset, route } = this.props; // prevent preventDefault for back(3) and forward(4) mouse buttons if (ev.button !== 3 && ev.button !== 4) { @@ -255,9 +263,8 @@ class Timeline extends Component { const endOffset = Math.round(this.percentToOffset(endPercent)); if (Math.abs(dragging[1] - dragging[0]) > 3) { - const offset = currentOffset(); if (offset < startOffset || offset > endOffset) { - this.props.dispatch(seek(startOffset)); + seekVideoPlayer(startOffset, route) } const { dispatch } = this.props; const startTime = startOffset; @@ -284,17 +291,23 @@ class Timeline extends Component { if (!this.mounted) { return; } - raf(this.getOffset); - let offset = currentOffset(); - if (this.seekIndex) { - offset = this.seekIndex; + let offset; + if (this.props.hasAudio && isIos()) { + // video with audio doesn't report currentTime properly so we must use onTimeUpdate reported time + offset = this.props.offset; + } else { + offset = getVideoPlayerCurrentTime(this.props.route); + if (offset === null) { + offset = this.props.offset; + } } - offset = Math.floor(offset); - const percent = this.offsetToPercent(offset); + let percent = this.offsetToPercent(offset); + if (percent >= 1) percent = 1; if (this.rulerRemaining.current && this.rulerRemaining.current.parentElement) { this.rulerRemaining.current.style.left = `${Math.floor(10000 * percent) / 100}%`; this.rulerRemaining.current.style.width = `${100 - Math.floor(10000 * percent) / 100}%`; } + this.rafId = raf(this.getOffset); } percentToOffset(perc) { @@ -447,8 +460,14 @@ class Timeline extends Component { } const stateToProps = Obstruction({ + offset: 'offset', zoom: 'zoom', loop: 'loop', + desiredPlaySpeed: 'desiredPlaySpeed', + isBufferingVideo: 'isBufferingVideo', + currentRoute: 'currentRoute', + isPlaying: 'isPlaying', + hasAudio: 'hasAudio', }); export default connect(stateToProps)(withStyles(styles)(Timeline)); diff --git a/src/components/explorer.jsx b/src/components/explorer.jsx index 7c43e3a31..f282a8a96 100644 --- a/src/components/explorer.jsx +++ b/src/components/explorer.jsx @@ -18,7 +18,6 @@ import BodyTeleop from './BodyTeleop'; import { analyticsEvent, selectDevice, updateDevices, checkLastRoutesData, streamNav } from '../actions'; import init from '../actions/startup'; import Colors from '../colors'; -import { play, pause } from '../timeline/playback'; import { verifyPairToken, pairErrorToMessage } from '../utils'; import ResizeHandler from './ResizeHandler'; @@ -141,19 +140,12 @@ class ExplorerApp extends Component { } componentDidUpdate(prevProps, prevState) { - const { pathname, zoom, dongleId, limit } = this.props; + const { pathname, dongleId, limit } = this.props; if (prevProps.pathname !== pathname) { this.setState({ drawerIsOpen: false }); } - if (!prevProps.zoom && zoom) { - this.props.dispatch(play()); - } - if (prevProps.zoom && !zoom) { - this.props.dispatch(pause()); - } - // this is necessary when user goes to explorer for the first time, dongleId is not populated in state yet // so init() will not successfully fetch routes data // when checkLastRoutesData is called within init(), it would set limit so we don't need to check again diff --git a/src/initialState.js b/src/initialState.js index 0e3a71cc2..6efe95717 100644 --- a/src/initialState.js +++ b/src/initialState.js @@ -15,8 +15,8 @@ export default { desiredPlaySpeed: 1, // speed set by user isBufferingVideo: true, // if we're currently buffering for more data + isPlaying: true, // if the video is currently playing offset: null, // in miliseconds, relative to state.zoom.start - startTime: Date.now(), // millisecond timestamp in which play began routes: null, routesMeta: { diff --git a/src/timeline/index.js b/src/timeline/index.js deleted file mode 100644 index 22724754f..000000000 --- a/src/timeline/index.js +++ /dev/null @@ -1,33 +0,0 @@ -import store from '../store'; - -/** - * Get current playback offset - * - * @param {object} state - * @returns {number} - */ -export function currentOffset(state = null) { - if (!state) { - state = store.getState(); - } - - /** @type {number} */ - let offset; - if (state.offset === null && state.loop?.startTime) { - offset = state.loop.startTime; - } else { - const playSpeed = state.isBufferingVideo ? 0 : state.desiredPlaySpeed; - offset = state.offset + ((Date.now() - state.startTime) * playSpeed); - } - - if (offset !== null && state.loop?.startTime) { - // respect the loop - const loopOffset = state.loop.startTime; - if (offset < loopOffset) { - offset = loopOffset; - } else if (offset > loopOffset + state.loop.duration) { - offset = ((offset - loopOffset) % state.loop.duration) + loopOffset; - } - } - return offset; -} \ No newline at end of file diff --git a/src/timeline/playback.js b/src/timeline/playback.js index acb381f5c..7d0aea0f6 100644 --- a/src/timeline/playback.js +++ b/src/timeline/playback.js @@ -1,47 +1,31 @@ -// basic helper functions for controlling playback -// we shouldn't want to edit the raw state most of the time, helper functions are better import * as Types from '../actions/types'; -import { currentOffset } from '.'; export function reducer(_state, action) { let state = { ..._state }; - let loopOffset = null; - if (state.loop && state.loop.startTime !== null) { - loopOffset = state.loop.startTime; - } switch (action.type) { case Types.ACTION_SEEK: state = { ...state, offset: action.offset, - startTime: Date.now(), }; - - if (loopOffset !== null) { - if (state.offset < loopOffset) { - state.offset = loopOffset; - } else if (state.offset > (loopOffset + state.loop.duration)) { - state.offset = loopOffset + state.loop.duration; - } - } break; - case Types.ACTION_PAUSE: + case Types.ACTION_PLAYBACK_SPEED: state = { ...state, - offset: currentOffset(state), - startTime: Date.now(), - desiredPlaySpeed: 0, + desiredPlaySpeed: action.speed, }; break; case Types.ACTION_PLAY: - if (action.speed !== state.desiredPlaySpeed) { - state = { - ...state, - offset: currentOffset(state), - desiredPlaySpeed: action.speed, - startTime: Date.now(), - }; - } + state = { + ...state, + isPlaying: true, + }; + break; + case Types.ACTION_PAUSE: + state = { + ...state, + isPlaying: false, + }; break; case Types.ACTION_LOOP: if (action.start !== null && action.start !== undefined && action.end !== null && action.end !== undefined) { @@ -57,51 +41,26 @@ export function reducer(_state, action) { state = { ...state, isBufferingVideo: action.buffering, - offset: currentOffset(state), - startTime: Date.now(), }; break; case Types.ACTION_RESET: state = { ...state, - desiredPlaySpeed: 1, - isBufferingVideo: true, offset: 0, - startTime: Date.now(), + desiredPlaySpeed: 1, + hasAudio: false, + }; + break; + case Types.ACTION_HAS_AUDIO: + state = { + ...state, + hasAudio: action.hasAudio, }; break; default: break; } - if (state.currentRoute && state.currentRoute.videoStartOffset && state.loop && state.zoom - && state.loop.startTime === state.zoom.start && state.zoom.start === 0) { - const loopRouteOffset = state.loop.startTime - state.zoom.start; - if (state.currentRoute.videoStartOffset > loopRouteOffset) { - state.loop = { - startTime: state.zoom.start + state.currentRoute.videoStartOffset, - duration: state.loop.duration - (state.currentRoute.videoStartOffset - loopRouteOffset), - }; - } - } - - // normalize over loop - if (state.offset !== null && state.loop?.startTime) { - const playSpeed = state.isBufferingVideo ? 0 : state.desiredPlaySpeed; - const offset = state.offset + (Date.now() - state.startTime) * playSpeed; - loopOffset = state.loop.startTime; - // has loop, trap offset within the loop - if (offset < loopOffset) { - state.startTime = Date.now(); - state.offset = loopOffset; - } else if (offset > loopOffset + state.loop.duration) { - state.offset = ((offset - loopOffset) % state.loop.duration) + loopOffset; - state.startTime = Date.now(); - } - } - - state.isBufferingVideo = Boolean(state.isBufferingVideo); - return state; } @@ -113,19 +72,20 @@ export function seek(offset) { }; } -// pause the playback -export function pause() { +// change playback speed without changing play/pause state +export function setPlaybackSpeed(speed) { return { - type: Types.ACTION_PAUSE, + type: Types.ACTION_PLAYBACK_SPEED, + speed, }; } -// resume / change play speed -export function play(speed = 1) { - return { - type: Types.ACTION_PLAY, - speed, - }; +export function play() { + return { type: Types.ACTION_PLAY }; +} + +export function pause() { + return { type: Types.ACTION_PAUSE }; } export function selectLoop(start, end) { @@ -149,3 +109,10 @@ export function resetPlayback() { type: Types.ACTION_RESET, }; } + +export function setHasAudio(hasAudio) { + return { + type: Types.ACTION_HAS_AUDIO, + hasAudio, + }; +} diff --git a/src/timeline/playback.test.js b/src/timeline/playback.test.js index f363f2083..08ccedc7c 100644 --- a/src/timeline/playback.test.js +++ b/src/timeline/playback.test.js @@ -1,7 +1,12 @@ /* eslint-env jest */ -import { asyncSleep } from '../utils'; -import { currentOffset } from '.'; -import { bufferVideo, pause, play, reducer, seek, selectLoop } from './playback'; +import { + bufferVideo, + reducer, + resetPlayback, + seek, + selectLoop, + setPlaybackSpeed, +} from './playback'; const makeDefaultStruct = function makeDefaultStruct() { return { @@ -13,104 +18,70 @@ const makeDefaultStruct = function makeDefaultStruct() { }; }; -// make Date.now super stable for tests -let mostRecentNow = Date.now(); -const oldNow = Date.now; -Date.now = function now() { - return mostRecentNow; -}; -function newNow() { - mostRecentNow = oldNow(); - return mostRecentNow; -} - describe('playback', () => { - it('has playback controls', async () => { - newNow(); + it('has playback controls', () => { let state = makeDefaultStruct(); - // should do nothing - state = reducer(state, pause()); + // stop playback + state = reducer(state, setPlaybackSpeed(0)); expect(state.desiredPlaySpeed).toEqual(0); - // start playing, should set start time and such - let playTime = newNow(); - state = reducer(state, play()); - // this is a (usually 1ms) race condition - expect(state.startTime).toEqual(playTime); + // start playing + state = reducer(state, setPlaybackSpeed(1)); expect(state.desiredPlaySpeed).toEqual(1); - await asyncSleep(100 + Math.random() * 200); - // should update offset - let ellapsed = newNow() - playTime; - state = reducer(state, pause()); - - expect(state.offset).toEqual(ellapsed); - - // start playing, should set start time and such - playTime = newNow(); - state = reducer(state, play(0.5)); - // this is a (usually 1ms) race condition - expect(state.startTime).toEqual(playTime); - expect(state.desiredPlaySpeed).toEqual(0.5); + // seek updates offset + state = reducer(state, seek(123)); + expect(state.offset).toEqual(123); - await asyncSleep(100 + Math.random() * 200); - // should update offset, playback speed 1/2 - ellapsed += (newNow() - playTime) / 2; - expect(currentOffset(state)).toEqual(ellapsed); - state = reducer(state, pause()); + // reset clears offset + state = reducer(state, resetPlayback()); + expect(state.offset).toEqual(0); + }); - expect(state.offset).toEqual(ellapsed); + it('should set loop start time and duration', () => { + let state = makeDefaultStruct(); - // seek! - newNow(); - state = reducer(state, seek(123)); - expect(state.offset).toEqual(123); - expect(state.startTime).toEqual(Date.now()); - expect(currentOffset(state)).toEqual(123); + state = reducer(state, selectLoop( + 1000, + 2000, + )); + expect(state.loop.startTime).toEqual(1000); + expect(state.loop.duration).toEqual(1000); }); - it('should clamp loop when seeked after loop end time', () => { - newNow(); + it('should not clamp offset when seeked after loop end time', () => { let state = makeDefaultStruct(); - // set up loop - state = reducer(state, play()); state = reducer(state, selectLoop( 1000, 2000, )); expect(state.loop.startTime).toEqual(1000); - // seek past loop end boundary a state = reducer(state, seek(3000)); expect(state.loop.startTime).toEqual(1000); - expect(state.offset).toEqual(2000); + expect(state.offset).toEqual(3000); }); - it('should clamp loop when seeked before loop start time', () => { - newNow(); + it('should not clamp offset when seeked before loop start time', () => { let state = makeDefaultStruct(); - // set up loop - state = reducer(state, play()); state = reducer(state, selectLoop( 1000, 2000, )); expect(state.loop.startTime).toEqual(1000); - // seek past loop end boundary a state = reducer(state, seek(0)); expect(state.loop.startTime).toEqual(1000); - expect(state.offset).toEqual(1000); + expect(state.offset).toEqual(0); }); - it('should buffer video and data', async () => { - newNow(); + it('should buffer video and data', () => { let state = makeDefaultStruct(); - state = reducer(state, play()); + state = reducer(state, setPlaybackSpeed(1)); expect(state.desiredPlaySpeed).toEqual(1); // claim the video is buffering @@ -118,17 +89,13 @@ describe('playback', () => { expect(state.desiredPlaySpeed).toEqual(1); expect(state.isBufferingVideo).toEqual(true); - state = reducer(state, play(0.5)); + state = reducer(state, setPlaybackSpeed(0.5)); expect(state.desiredPlaySpeed).toEqual(0.5); expect(state.isBufferingVideo).toEqual(true); - expect(state.desiredPlaySpeed).toEqual(0.5); - - state = reducer(state, play(2)); + state = reducer(state, setPlaybackSpeed(2)); state = reducer(state, bufferVideo(false)); expect(state.desiredPlaySpeed).toEqual(2); expect(state.isBufferingVideo).toEqual(false); - - expect(state.desiredPlaySpeed).toEqual(2); }); -}); \ No newline at end of file +}); diff --git a/src/timeline/segments.test.js b/src/timeline/segments.test.js index edb586715..70a824c21 100644 --- a/src/timeline/segments.test.js +++ b/src/timeline/segments.test.js @@ -29,7 +29,7 @@ const routes = [{ describe('segments', () => { it('finds current segment', async () => { const [route] = routes; - expect(getSegmentNumber(route)).toBe(0); + expect(getSegmentNumber(route, 0)).toBe(0); }); it('can check if it has segment metadata', () => { @@ -97,4 +97,4 @@ describe('segments', () => { dongleId: 'asdfasdf', })).toBe(true); }); -}); \ No newline at end of file +}); diff --git a/src/timeline/videoPlayer.js b/src/timeline/videoPlayer.js new file mode 100644 index 000000000..15da52214 --- /dev/null +++ b/src/timeline/videoPlayer.js @@ -0,0 +1,75 @@ +let videoPlayer = null; + +export function setVideoPlayer(player) { + videoPlayer = player; +} + +function getInternal() { + if (!videoPlayer || !videoPlayer.getInternalPlayer) { + return null; + } + return videoPlayer.getInternalPlayer(); +} + +export function seekVideoPlayer(offset, route) { + if (!videoPlayer || !videoPlayer.getInternalPlayer || !videoPlayer.getDuration()) { + return false; + } + const internal = getInternal(); + + let videoTime = offset; + if (route && route.videoStartOffset) { + videoTime -= route.videoStartOffset; + } + videoTime = Math.max(0, videoTime / 1000); + + internal.currentTime = videoTime; + return true; +} + +export function getVideoPlayerCurrentTime(route) { + const internal = getInternal(); + if (!internal) { + return null; + } + const videoStartOffset = (route && route.videoStartOffset) || 0; + return internal.currentTime * 1000 + videoStartOffset; +} + +export function isVideoPaused() { + const internal = getInternal(); + if (!internal) { + return true; + } + return internal.paused; +} + +export function playVideo() { + const internal = getInternal(); + if (!internal) { + return false; + } + const promise = internal.play(); + if (promise && typeof promise.catch === 'function') { + promise.catch(() => {}); + } + return true; +} + +export function pauseVideo() { + const internal = getInternal(); + if (!internal) { + return false; + } + internal.pause(); + return true; +} + +export function setVideoPlaybackRate(rate) { + const internal = getInternal(); + if (!internal) { + return false; + } + internal.playbackRate = rate; + return true; +} diff --git a/src/utils/browser.js b/src/utils/browser.js index cd4030e57..35dbbf79b 100644 --- a/src/utils/browser.js +++ b/src/utils/browser.js @@ -1,7 +1,3 @@ export function isIos() { return /iphone|ipad|ipod/i.test(navigator.userAgent); } - -export function isFirefox() { - return navigator.userAgent.toLowerCase().includes('firefox'); -} diff --git a/src/utils/index.js b/src/utils/index.js index ac8b67171..f1c560fa0 100644 --- a/src/utils/index.js +++ b/src/utils/index.js @@ -3,8 +3,6 @@ import dayjs from 'dayjs'; import relativeTime from 'dayjs/plugin/relativeTime'; import decodeJwt, { InvalidTokenError } from 'jwt-decode'; -import { currentOffset } from '../timeline'; - dayjs.extend(relativeTime); export const emptyDevice = { @@ -193,8 +191,8 @@ export function getSegmentNumber(route, offset) { if (!route) { return null; } - if (offset === undefined) { - offset = currentOffset(); + if (offset === undefined || offset === null) { + return null; } return Math.floor(offset / (60*1000));