Consume background thread movie decoding.

src/Banner.cpp

@@ -43,7 +43,6 @@ void Banner::Load( RageTextureID ID, bool bIsBanner )
 	m_bScrolling = false;
 
 	TEXTUREMAN->DisableOddDimensionWarning();
-	TEXTUREMAN->VolatileTexture( ID );
 	Sprite::Load( ID );
 	TEXTUREMAN->EnableOddDimensionWarning();
 };

src/arch/MovieTexture/MovieTexture_FFMpeg.cpp

@@ -124,11 +124,6 @@ MovieDecoder_FFMpeg::MovieDecoder_FFMpeg()
 
 MovieDecoder_FFMpeg::~MovieDecoder_FFMpeg()
 {
-	if( m_iCurrentPacketOffset != -1 )
-	{
-		avcodec::av_packet_unref( &m_Packet );
-		m_iCurrentPacketOffset = -1;
-	}
 	if (m_swsctx)
 	{
 		avcodec::sws_freeContext(m_swsctx);
@@ -156,19 +151,13 @@ void MovieDecoder_FFMpeg::Init()
 	m_iEOF = 0;
 	m_fTimestamp = 0;
 	m_fLastFrameDelay = 0;
-	m_iFrameNumber = -1; /* decode one frame and you're on the 0th */
+	m_iFrameNumber = 0;
 	m_totalFrames = 0;
 	m_fTimestampOffset = 0;
 	m_fLastFrame = 0;
 	m_swsctx = nullptr;
 	m_avioContext = nullptr;
 	m_buffer = nullptr;
-
-	if( m_iCurrentPacketOffset != -1 )
-	{
-		avcodec::av_packet_unref( &m_Packet );
-		m_iCurrentPacketOffset = -1;
-	}
 }
 
 /* Read until we get a frame, EOF or error.  Return -1 on error, 0 on EOF, 1 if we have a frame. */
@@ -211,10 +200,28 @@ int MovieDecoder_FFMpeg::DecodeFrame( float fTargetTime )
 
 float MovieDecoder_FFMpeg::GetTimestamp() const
 {
-	return m_fTimestamp - m_fTimestampOffset;
+	// Always display the first frame.
+	if (m_iFrameNumber == 0) {
+		return 0;
+	}
+
+	// In a logical situation, this means that display is outpacing decoding.
+	if (m_iFrameNumber >= m_FrameBuffer.size()) {
+		return 0;
+	}
+	return m_FrameBuffer[m_iFrameNumber].frameTimestamp;
 }
 
 bool MovieDecoder_FFMpeg::IsCurrentFrameReady() {
+	// We're displaying faster than decoding. Do not even try to display the frame.
+	if (m_iFrameNumber >= m_FrameBuffer.size()) {
+		return false;
+	}
+	// If the whole movie is decoded, then the frame is definitely ready.
+	if (m_iEOF) {
+		return true;
+	}
+
 	std::lock_guard<std::mutex>(m_FrameBuffer[m_iFrameNumber].lock);
 	if (!m_FrameBuffer[m_iFrameNumber].decoded) {
 		LOG->Info("Frame %i not decoded, total frames: %i", m_iFrameNumber, m_totalFrames);
@@ -503,32 +510,41 @@ int MovieDecoder_FFMpeg::DecodePacket( float fTargetTime )
 	return 0; /* packet done */
 }
 
-void MovieDecoder_FFMpeg::GetFrame( RageSurface *pSurface )
+bool MovieDecoder_FFMpeg::GetFrame(RageSurface* pSurface)
 {
 	avcodec::AVFrame pict;
-	pict.data[0] = (unsigned char *) pSurface->pixels;
+	pict.data[0] = (unsigned char*)pSurface->pixels;
 	pict.linesize[0] = pSurface->pitch;
 
 	/* XXX 1: Do this in one of the Open() methods instead?
 	 * XXX 2: The problem of doing this in Open() is that m_AVTexfmt is not
 	 * already initialized with its correct value.
 	 */
-	if( m_swsctx == nullptr )
+	if (m_swsctx == nullptr)
 	{
-		m_swsctx = avcodec::sws_getCachedContext( m_swsctx,
-				GetWidth(), GetHeight(), m_pStreamCodec->pix_fmt,
-				GetWidth(), GetHeight(), m_AVTexfmt,
-				sws_flags, nullptr, nullptr, nullptr );
-		if( m_swsctx == nullptr )
+		m_swsctx = avcodec::sws_getCachedContext(m_swsctx,
+			GetWidth(), GetHeight(), m_pStreamCodec->pix_fmt,
+			GetWidth(), GetHeight(), m_AVTexfmt,
+			sws_flags, nullptr, nullptr, nullptr);
+		if (m_swsctx == nullptr)
 		{
 			LOG->Warn("Cannot initialize sws conversion context for (%d,%d) %d->%d", GetWidth(), GetHeight(), m_pStreamCodec->pix_fmt, m_AVTexfmt);
-			return;
+			return false;
 		}
 	}
 
-	avcodec::sws_scale( m_swsctx,
-			m_Frame->data, m_Frame->linesize, 0, GetHeight(),
-			pict.data, pict.linesize );
+	avcodec::sws_scale(m_swsctx,
+		m_FrameBuffer[m_iFrameNumber].frame.data, m_FrameBuffer[m_iFrameNumber].frame.linesize, 0, GetHeight(),
+		pict.data, pict.linesize);
+
+	// Don't advance the frame number past the (potential) end of the buffer.
+	// This can happen if display is outpacing decoding, or if we're at the
+	// end of file.
+	if (m_iFrameNumber >= (m_totalFrames - 1)) {
+		return m_iEOF;
+	}
+	m_iFrameNumber++;
+	return false;
 }
 
 static RString averr_ssprintf( int err, const char *fmt, ... )
@@ -677,8 +693,7 @@ void MovieDecoder_FFMpeg::Close()
 
 void MovieDecoder_FFMpeg::Rewind()
 {
-	avcodec::av_seek_frame( m_fctx, -1, 0, 0 );
-	OpenCodec();
+	m_iFrameNumber = 0;
 }
 
 RageSurface *MovieDecoder_FFMpeg::CreateCompatibleSurface( int iTextureWidth, int iTextureHeight, bool bPreferHighColor, MovieDecoderPixelFormatYCbCr &fmtout )

src/arch/MovieTexture/MovieTexture_FFMpeg.h

@@ -70,7 +70,9 @@ public:
 	void Close();
 	void Rewind();
 
-	void GetFrame( RageSurface *pOut );
+	// This draws a frame from the buffer onto the provided RageSurface.
+	// Returns true if returning the last frame in the movie.
+	bool GetFrame(RageSurface* pOut);
 	int DecodeFrame( float fTargetTime );
 
 	// Decode a single frame.  Return -2 on cancel, -1 on error, 0 on EOF, 1 if we have a frame.

src/arch/MovieTexture/MovieTexture_Generic.cpp

@@ -6,6 +6,7 @@
 #include "RageSurface.h"
 #include "RageTextureManager.h"
 #include "RageTextureRenderTarget.h"
+#include "RageTimer.h"
 #include "RageUtil.h"
 #include "Sprite.h"
 
@@ -43,30 +44,38 @@ MovieTexture_Generic::MovieTexture_Generic( RageTextureID ID, MovieDecoder *pDec
 
 RString MovieTexture_Generic::Init()
 {
-	RString sError = m_pDecoder->Open( GetID().filename );
-	if( sError != "" )
+	RString sError = m_pDecoder->Open(GetID().filename);
+	if (sError != "")
 		return sError;
 
 	CreateTexture();
 	CreateFrameRects();
 
-	/* Decode one frame, to guarantee that the texture is drawn when this function returns. */
-	int ret = m_pDecoder->DecodeFrame( -1 );
-	if( ret == -1 )
-		return ssprintf( "%s: error getting first frame", GetID().filename.c_str() );
-	if( ret == 0 )
-	{
-		/* There's nothing there. */
-		return ssprintf( "%s: EOF getting first frame", GetID().filename.c_str() );
-	}
 
-	m_ImageWaiting = FRAME_DECODED;
+	// Draw the first frame immediately, to guarantee that the texture is drawn
+	// when this function returns--if possible.
+	if (m_pDecoder->DecodeNextFrame() < 0) {
+		LOG->Trace("Failure to decode first frame of video file \"%s\"", GetID().filename.c_str());
+		m_failure = true;
+		return RString("Failure to display movie.");
+	};
+	UpdateMovie(0);
 
-	LOG->Trace( "Resolution: %ix%i (%ix%i, %ix%i)",
-			m_iSourceWidth, m_iSourceHeight,
-			m_iImageWidth, m_iImageHeight, m_iTextureWidth, m_iTextureHeight );
+	decoding_thread = std::make_unique<std::thread>([this]() {
+		LOG->Trace("Beginning to decode video file \"%s\"", GetID().filename.c_str());
+		auto timer = RageTimer();
 
-	UpdateFrame();
+		int ret = m_pDecoder->DecodeMovie();
+		if (ret == -1) {
+			m_failure = true;
+		}
+
+		LOG->Trace("Done decoding video file \"%s\", took %f seconds", GetID().filename.c_str(), timer.Ago());
+		});
+
+	LOG->Trace("Resolution: %ix%i (%ix%i, %ix%i)",
+		m_iSourceWidth, m_iSourceHeight,
+		m_iImageWidth, m_iImageHeight, m_iTextureWidth, m_iTextureHeight);
 
 	CHECKPOINT_M("Generic initialization completed. No errors found.");
 
@@ -75,8 +84,11 @@ RString MovieTexture_Generic::Init()
 
 MovieTexture_Generic::~MovieTexture_Generic()
 {
-	if( m_pDecoder )
+	if (m_pDecoder) {
+		m_pDecoder->Cancel();
+		decoding_thread->join();
 		m_pDecoder->Close();
+	}
 
 	/* m_pSprite may reference the texture; delete it before DestroyTexture. */
 	delete m_pSprite;
@@ -361,88 +373,33 @@ bool MovieTexture_Generic::DecodeFrame()
 
 /*
  * Returns:
- *  == 0 if the currently decoded frame is ready to be displayed
- *   > 0 (seconds) if it's not yet time to display;
+ *  <= 0 if it's time for the next frame to display
+ *   > 0 (seconds) if it's not yet time to display
  */
 float MovieTexture_Generic::CheckFrameTime()
 {
-	if( m_fRate == 0 )
+	if (m_fRate == 0) {
 		return 1;	// "a long time until the next frame"
-
-	const float fOffset = (m_pDecoder->GetTimestamp() - m_fClock) / m_fRate;
-
-	/* If we're ahead, we're decoding too fast; delay. */
-	if( fOffset > 0.00001f )
-	{
-		if( m_bFrameSkipMode )
-		{
-			/* We're caught up; stop skipping frames. */
-			LOG->Trace( "stopped skipping frames" );
-			m_bFrameSkipMode = false;
-		}
-		return fOffset;
 	}
-
-	/*
-	 * We're behind by -Offset seconds.
-	 *
-	 * If we're just slightly behind, don't worry about it; we'll simply
-	 * not sleep, so we'll move as fast as we can to catch up.
-	 *
-	 * If we're far behind, we're short on CPU.  Skip texture updates; this
-	 * is a big bottleneck on many systems.
-	 *
-	 * If we hit a threshold, start skipping frames via #1.  If we do that,
-	 * don't stop once we hit the threshold; keep doing it until we're fully
-	 * caught up.
-	 *
-	 * We should try to notice if we simply don't have enough CPU for the video;
-	 * it's better to just stay in frame skip mode than to enter and exit it
-	 * constantly, but we don't want to do that due to a single timing glitch.
-	 */
-	const float FrameSkipThreshold = 0.5f;
-
-	if( -fOffset >= FrameSkipThreshold && !m_bFrameSkipMode )
-	{
-		LOG->Trace( "(%s) Time is %f, and the movie is at %f.  Entering frame skip mode.",
-			GetID().filename.c_str(), m_fClock, m_pDecoder->GetTimestamp() );
-		m_bFrameSkipMode = true;
-	}
-
-	return 0;
+	return (m_pDecoder->GetTimestamp() - m_fClock) / m_fRate;
 }
 
-/* Decode data. */
-void MovieTexture_Generic::UpdateMovie( float fSeconds )
+void MovieTexture_Generic::UpdateMovie(float fSeconds)
 {
-	m_fClock += fSeconds * m_fRate;
-
-	/* We might need to decode more than one frame per update.  However, there
-	 * have been bugs in ffmpeg that cause it to not handle EOF properly, which
-	 * could make this never return, so let's play it safe. */
-	int iMax = 4;
-	while( --iMax )
-	{
-		/* If we don't have a frame decoded, decode one. */
-		if( m_ImageWaiting == FRAME_NONE )
-		{
-			if( !DecodeFrame() )
-				break;
-
-			m_ImageWaiting = FRAME_DECODED;
-		}
-
-		/* If we have a frame decoded, see if it's time to display it. */
-		float fTime = CheckFrameTime();
-		if ( fTime <= 0 )
-		{
-			UpdateFrame();
-			m_ImageWaiting = FRAME_NONE;
-		}
+	// Quick exit in case we failed to decode the movie.
+	if (m_failure) {
 		return;
 	}
+	m_fClock += fSeconds * m_fRate;
 
-	LOG->MapLog( "movie_looping", "MovieTexture_Generic::Update looping" );
+	// If the frame isn't ready, don't update. This does mean the video
+	// will "speed up" to catch up when decoding does outpace display.
+	//
+	// In practice, display should rarely, if ever, outpace decoding.
+	if (m_pDecoder->IsCurrentFrameReady() && CheckFrameTime() <= 0) {
+		UpdateFrame();
+		return;
+	}
 }
 
 void MovieTexture_Generic::UpdateFrame()
@@ -450,42 +407,54 @@ void MovieTexture_Generic::UpdateFrame()
 	/* Just in case we were invalidated: */
 	CreateTexture();
 
-	if( m_pTextureLock != nullptr )
+	if(m_pTextureLock != nullptr)
 	{
-		std::uintptr_t iHandle = m_pTextureIntermediate != nullptr? m_pTextureIntermediate->GetTexHandle(): this->GetTexHandle();
-		m_pTextureLock->Lock( iHandle, m_pSurface );
+		std::uintptr_t iHandle = m_pTextureIntermediate != nullptr ? m_pTextureIntermediate->GetTexHandle(): this->GetTexHandle();
+		m_pTextureLock->Lock(iHandle, m_pSurface);
 	}
 
-	m_pDecoder->GetFrame( m_pSurface );
-	if( m_pTextureLock != nullptr )
-		m_pTextureLock->Unlock( m_pSurface, true );
+	/* Are we looping? */
+	if (m_pDecoder->GetFrame(m_pSurface) && m_bLoop) {
+		LOG->Trace("File \"%s\" looping", GetID().filename.c_str());
+		m_pDecoder->Rewind();
+		// There's a gap in the audio when the music preview loops. This value
+		// is dynamic based on the ending and starting beats (see
+		// GameSoundManager.cpp::StartMusic).
+		//
+		// This means that the video will be off-sync during the loop, since
+		// the movie texture doesn't have access to the SoundManager's offset.
+		// Until it does, we can either freeze at the end of the video banner,
+		// or give it a best effort approximation (0.5 seconds).
+		m_fClock = 0.5;
+	};
 
-	if( m_pRenderTarget != nullptr )
+	if (m_pTextureLock != nullptr) {
+		m_pTextureLock->Unlock(m_pSurface, true);
+	}
+
+	if (m_pRenderTarget != nullptr)
 	{
-		CHECKPOINT_M( "About to upload the texture.");
+		CHECKPOINT_M("About to upload the texture.");
 
 		/* If we have no m_pTextureLock, we still have to upload the texture. */
-		if( m_pTextureLock == nullptr )
-		{
+		if (m_pTextureLock == nullptr) {
 			DISPLAY->UpdateTexture(
 				m_pTextureIntermediate->GetTexHandle(),
 				m_pSurface,
 				0, 0,
-				m_pSurface->w, m_pSurface->h );
+				m_pSurface->w, m_pSurface->h);
 		}
-		m_pRenderTarget->BeginRenderingTo( false );
+		m_pRenderTarget->BeginRenderingTo(false);
 		m_pSprite->Draw();
 		m_pRenderTarget->FinishRenderingTo();
 	}
-	else
-	{
-		if( m_pTextureLock == nullptr )
-		{
+	else {
+		if (m_pTextureLock == nullptr) {
 			DISPLAY->UpdateTexture(
 				m_uTexHandle,
 				m_pSurface,
 				0, 0,
-				m_iImageWidth, m_iImageHeight );
+				m_iImageWidth, m_iImageHeight);
 		}
 	}
 }
@@ -506,19 +475,19 @@ void MovieTexture_Generic::Reload()
 {
 }
 
-void MovieTexture_Generic::SetPosition( float fSeconds )
+void MovieTexture_Generic::SetPosition(float fSeconds)
 {
-	/* We can reset to 0, but I don't think this API supports fast seeking
-	 * yet.  I don't think we ever actually seek except to 0 right now,
-	 * anyway. XXX */
-	if( fSeconds != 0 )
+	// In theory, we can math out fSeconds and frame counts to seek the video,
+	// but there's likely no practical use case of this.
+	if (fSeconds != 0)
 	{
-		LOG->Warn( "MovieTexture_Generic::SetPosition(%f): non-0 seeking unsupported; ignored", fSeconds );
+		LOG->Warn("MovieTexture_Generic::SetPosition(%f): non-0 seeking unsupported; ignored", fSeconds);
 		return;
 	}
 
-	LOG->Trace( "Seek to %f", fSeconds );
-	m_bWantRewind = true;
+	LOG->Trace("Seek to %f", fSeconds);
+	m_fClock = 0;
+	m_pDecoder->Rewind();
 }
 
 std::uintptr_t MovieTexture_Generic::GetTexHandle() const

src/arch/MovieTexture/MovieTexture_Generic.h

@@ -4,6 +4,7 @@
 #include "MovieTexture.h"
 
 #include <cstdint>
+#include <thread>
 
 class FFMpeg_Helper;
 struct RageSurface;
@@ -51,7 +52,7 @@ public:
 	/*
 	 * Get the currently-decoded frame.
 	 */
-	virtual void GetFrame( RageSurface *pOut ) = 0;
+	virtual bool GetFrame( RageSurface *pOut ) = 0;
 
 	/* Return the dimensions of the image, in pixels (before aspect ratio
 	 * adjustments). */
@@ -117,12 +118,17 @@ public:
 private:
 	MovieDecoder *m_pDecoder;
 
+	std::unique_ptr<std::thread> decoding_thread;
+
 	float m_fRate;
 	enum {
 		FRAME_NONE, /* no frame available; call GetFrame to get one */
 		FRAME_DECODED /* frame decoded; waiting until it's time to display it */
 	} m_ImageWaiting;
 	bool m_bLoop;
+
+	// If true, halts all decoding and display.
+	bool m_failure = false;
 	bool m_bWantRewind;
 
 	enum State { DECODER_QUIT, DECODER_RUNNING } m_State;