Pass the surface directly to GetFrame, and a timestamp if frame skipping is

enabled.
 - This is simpler--two fewer entry points, and a trivial decode path.
 - Puts the mechanics of frame skipping in the decoder's control.  Previously,
   all we did was skip conversion and texture updates.  Now, the decoder
   can be smarter: skip B-frames, or skip to a keyframe.
 - This allows for faster decodes, using slice-based conversion: YUV conversion
   can be done at the same time as decoding.

stepmania/src/arch/MovieTexture/MovieTexture_FFMpeg.cpp

@@ -164,9 +164,7 @@ public:
 	CString Open( CString sFile );
 	void Close();
 
-	int GetFrame();
+	int GetFrame( RageSurface *pOut, float fTargetTime );
-
-	void ConvertToSurface( RageSurface *pSurface ) const;
 
 	int GetWidth() const { return m_pStream->codec.width; }
 	int GetHeight() const { return m_pStream->codec.height; }
@@ -176,12 +174,11 @@ public:
 	float GetTimestamp() const;
 	float GetFrameDuration() const;
 
-	bool SkippableFrame() const { return (m_pStream->codec.frame_number % 2) == 0; }
-
 private:
 	void Init();
 	int ReadPacket();
 	int DecodePacket();
+	void ConvertToSurface( RageSurface *pSurface ) const;
 
 	avcodec::AVStream *m_pStream;
 	avcodec::AVFrame m_Frame;
@@ -241,8 +238,9 @@ void MovieDecoder_FFMpeg::Init()
 }
 
 /* Read until we get a frame, EOF or error.  Return -1 on error, 0 on EOF, 1 if we have a frame. */
-int MovieDecoder_FFMpeg::GetFrame()
+int MovieDecoder_FFMpeg::GetFrame( RageSurface *pOut, float fTargetTime )
 {
+restart:
 	while( 1 )
 	{
 		int ret = DecodePacket();
@@ -279,6 +277,13 @@ int MovieDecoder_FFMpeg::GetFrame()
 		}
 	}
 
+	if( fTargetTime != -1 &&
+		GetTimestamp() + GetFrameDuration() <= fTargetTime &&
+		(m_pStream->codec.frame_number % 2) == 0 )
+		goto restart;
+
+	ConvertToSurface( pOut );
+
 	return 1;
 }
 

stepmania/src/arch/MovieTexture/MovieTexture_Generic.cpp

@@ -38,8 +38,11 @@ CString MovieTexture_Generic::Init()
 	if( sError != "" )
 		return sError;
 
+	CreateTexture();
+	CreateFrameRects();
+
 	/* Decode one frame, to guarantee that the texture is drawn when this function returns. */
-	int ret = m_pDecoder->GetFrame();
+	int ret = m_pDecoder->GetFrame( m_pSurface, -1 );
 	if( ret == -1 )
 		return ssprintf( "%s: error getting first frame", GetID().filename.c_str() );
 	if( ret == 0 )
@@ -48,16 +51,12 @@ CString MovieTexture_Generic::Init()
 		return ssprintf( "%s: EOF getting first frame", GetID().filename.c_str() );
 	}
 
-	m_ImageWaiting = FRAME_DECODED;
+	m_ImageWaiting = FRAME_WAITING;
 
-	CreateTexture();
 	LOG->Trace( "Resolution: %ix%i (%ix%i, %ix%i)",
 			m_iSourceWidth, m_iSourceHeight,
 			m_iImageWidth, m_iImageHeight, m_iTextureWidth, m_iTextureHeight );
 
-	CreateFrameRects();
-
-	ConvertFrame();
 	UpdateFrame();
 
 	CHECKPOINT;
@@ -198,7 +197,11 @@ bool MovieTexture_Generic::DecodeFrame()
 		CHECKPOINT;
 
 		/* Read a frame. */
-		int ret = m_pDecoder->GetFrame();
+		float fTargetTime = -1;
+		if( m_bFrameSkipMode && m_fClock > m_pDecoder->GetTimestamp() )
+			fTargetTime = m_fClock;
+
+		int ret = m_pDecoder->GetFrame( m_pSurface, fTargetTime );
 		if( ret == -1 )
 			return false;
 
@@ -228,7 +231,6 @@ bool MovieTexture_Generic::DecodeFrame()
  * Returns:
  *  == 0 if the currently decoded frame is ready to be displayed
  *   > 0 (seconds) if it's not yet time to display;
- *  == -1 if we're behind and the frame should be skipped
  */
 float MovieTexture_Generic::CheckFrameTime()
 {
@@ -277,9 +279,6 @@ float MovieTexture_Generic::CheckFrameTime()
 		m_bFrameSkipMode = true;
 	}
 
-	if( m_bFrameSkipMode && m_pDecoder->SkippableFrame() )
-		return -1; /* skip */
-	
 	return 0;
 }
 
@@ -314,11 +313,7 @@ void MovieTexture_Generic::DecoderThread()
 		}
 
 		const float fTime = CheckFrameTime();
-		if( fTime == -1 )	// skip frame
+		if( fTime > 0 )		// not time to decode a new frame yet
-		{
-			DiscardFrame();
-		}
-		else if( fTime > 0 )		// not time to decode a new frame yet
 		{
 			/* This needs to be relatively short so that we wake up quickly 
 			 * from being paused or for changes in m_fRate. */
@@ -332,7 +327,8 @@ void MovieTexture_Generic::DecoderThread()
 				int n = m_BufferFinished.GetValue();
 				ASSERT_M( n == 0 || m_State == DECODER_QUIT, ssprintf("%i, %i", n, m_State) );
 			}
-			ConvertFrame();
+
+			m_ImageWaiting = FRAME_WAITING;
 
 			/* We just went into FRAME_WAITING.  Don't actually check; the main thread
 			 * will change us back to FRAME_NONE without locking, and poke m_BufferFinished.
@@ -367,10 +363,8 @@ void MovieTexture_Generic::Update(float fDeltaTime)
 				float fTime = CheckFrameTime();
 				if( fTime > 0 )
 					return;
-				else if( fTime == -1 )
-					DiscardFrame();
 				else
-					ConvertFrame();
+					m_ImageWaiting = FRAME_WAITING;
 			}
 		}
 
@@ -390,17 +384,6 @@ void MovieTexture_Generic::Update(float fDeltaTime)
 	LOG->MapLog( "movie_looping", "MovieTexture_Generic::Update looping" );
 }
 
-/* Convert the frame from the native (typically YUV) internal format to
- * our RGB RageSurface. */
-void MovieTexture_Generic::ConvertFrame()
-{
-	ASSERT_M( m_ImageWaiting == FRAME_DECODED, ssprintf("%i", m_ImageWaiting ) );
-
-	m_pDecoder->ConvertToSurface( m_pSurface );
-
-	m_ImageWaiting = FRAME_WAITING;
-}
-
 /* Call from the main thread when m_ImageWaiting == FRAME_WAITING to update the
  * texture.  Sets FRAME_NONE.  Does not signal m_BufferFinished. */
 void MovieTexture_Generic::UpdateFrame()

stepmania/src/arch/MovieTexture/MovieTexture_Generic.h

@@ -15,12 +15,17 @@ public:
 	virtual CString Open( CString sFile ) = 0;
 	virtual void Close() = 0;
 
-	/* Decode a frame internally. */
+	/*
-	virtual int GetFrame() = 0;
+	 * Decode a frame.  Return 1 on success, 0 on EOF, -1 on fatal error.
-
+	 *
-	/* Convert the current frame to an RGB surface.  This may be skipped
+	 * If we're lagging behind the video, fTargetTime will be the target
-	 * in frame skip mode. */
+	 * timestamp.  The decoder may skip frames to catch up.  On return,
-	virtual void ConvertToSurface( RageSurface *pSurface ) const = 0;
+	 * the current timestamp must be <= fTargetTime.
+	 *
+	 * Otherwise, fTargetTime will be -1, and the next frame should be
+	 * decoded; skip frames only if necessary to recover from errors.
+	 */
+	virtual int GetFrame( RageSurface *pOut, float fTargetTime ) = 0;
 
 	/* Return the dimensions of the image, in pixels (before aspect ratio
 	 * adjustments). */
@@ -30,12 +35,11 @@ public:
 	/* Return the aspect ratio of a pixel in the image.  Usually 1. */
 	virtual float GetSourceAspectRatio() const { return 1.0f; }
 
-	/* Create a surface.  This must be compatible with ConvertToSurface,
+	/* Create a surface acceptable to pass to GetFrame.  This should be
-	 * should be a surface which is realtime-compatible with DISPLAY, and
+	 * a surface which is realtime-compatible with DISPLAY, and should
-	 * should attempt to obey TEXTUREMAN->GetPrefs().m_iMovieColorDepth.
+	 * attempt to obey bPreferHighColor.  The given size will usually be
-	 * The given size will usually be the next power of two higher than
+	 * the next power of two higher than GetWidth/GetHeight, but on systems
-	 * GetWidth/GetHeight, but on systems with limited texture resolution,
+	 * with limited texture resolution, may be smaller. */
-	 * may be smaller. */
 	virtual RageSurface *CreateCompatibleSurface( int iTextureWidth, int iTextureHeight, bool bPreferHighColor ) = 0;
 
 	/* The following functions return information about the current frame,
@@ -48,11 +52,6 @@ public:
 
 	/* Get the duration, in seconds, to display the current frame. */
 	virtual float GetFrameDuration() const = 0;
-
-	/* Ugly: return true if, when in frame skip mode, this frame should be skipped.
-	 * Typically returns true on even frames, false on odd frames.  (Frame skip mode
-	 * should be smarter, and skip to a keyframe.) */
-	virtual bool SkippableFrame() const = 0;
 };
 
 
@@ -81,8 +80,8 @@ private:
 	float m_fRate;
 	enum {
 		FRAME_NONE, /* no frame available; call GetFrame to get one */
-		FRAME_DECODED, /* frame decoded; call ConvertFrame */
+		FRAME_DECODED, /* frame decoded; waiting until it's time to display it */
-		FRAME_WAITING /* frame converted and waiting to be uploaded */
+		FRAME_WAITING /* frame waiting to be uploaded */
 	} m_ImageWaiting;
 	bool m_bLoop;
 	bool m_bWantRewind;
@@ -115,7 +114,6 @@ private:
 	void DecoderThread();
 	RageThread m_DecoderThread;
 
-	void ConvertFrame();
 	void UpdateFrame();
 
 	void CreateTexture();

stepmania/src/arch/MovieTexture/MovieTexture_Theora.cpp

@@ -28,9 +28,8 @@ public:
 	CString Open( CString sFile );
 	void Close();
 
-	int GetFrame();
+	int GetFrame( RageSurface *pOut, float fTargetTime );
 	RageSurface *CreateCompatibleSurface( int iTextureWidth, int iTextureHeight, bool bPreferHighColor );
-	void ConvertToSurface( RageSurface *pSurface ) const;
 
 	int GetWidth() const { return m_TheoraInfo.frame_width; }
 	int GetHeight() const { return m_TheoraInfo.frame_height; }
@@ -38,12 +37,12 @@ public:
 
 	float GetTimestamp() const;
 	float GetFrameDuration() const;
-	bool SkippableFrame() const;
 
 private:
 	void Init();
 	CString ProcessHeaders();
 	int ReadPage( ogg_page *pOggPage );
+	void ConvertToSurface( RageSurface *pSurface ) const;
 
 	RageFile m_File;
 
@@ -215,7 +214,7 @@ CString MovieDecoder_Theora::Open( CString sFile )
 	return CString();
 }
 
-int MovieDecoder_Theora::GetFrame()
+int MovieDecoder_Theora::GetFrame( RageSurface *pOut, float fTargetTime )
 {
 	while(1)
 	{
@@ -223,6 +222,16 @@ int MovieDecoder_Theora::GetFrame()
 		if( ogg_stream_packetout(&m_OggStream, &op) != 0 )
 		{
 			theora_decode_packetin( &m_TheoraState, &op );
+
+			if( fTargetTime != -1 &&
+				GetTimestamp() + GetFrameDuration() <= fTargetTime )
+			{
+				ogg_int64_t iFrame = theora_granule_frame( (theora_state *) &m_TheoraState, m_iGranulepos );
+				if( (iFrame % 2) == 0 )
+					continue;
+			}
+
+			ConvertToSurface( pOut );
 			m_iGranulepos = m_TheoraState.granulepos;
 			return 1;
 		}
@@ -294,13 +303,6 @@ float MovieDecoder_Theora::GetFrameDuration() const
 	return (float) m_TheoraInfo.fps_denominator / m_TheoraInfo.fps_numerator;
 }
 
-bool MovieDecoder_Theora::SkippableFrame() const
-{
-	ogg_int64_t iFrame = theora_granule_frame( (theora_state *) &m_TheoraState, m_iGranulepos );
-	return (iFrame % 2) == 0;
-}
-
-
 void MovieDecoder_Theora::Close()
 {
 	ogg_stream_clear( &m_OggStream );