Fix D3D z-buffer problem by having separate GetOrthoMatrix functions for OGL and D3D.

I don't understand why the two APIs create an ortho matrix so differently, but I'm tried of messing with it for now.

stepmania/src/RageDisplay.cpp

@@ -298,7 +298,7 @@ void RageDisplay::LoadMenuPerspective(float fovDegrees)
 	if( fovDegrees == 0 )
 	{
  		float left = 0, right = SCREEN_WIDTH, bottom = SCREEN_HEIGHT, top = 0;
-		g_ProjectionStack.LoadMatrix( RageOrtho(left, right, bottom, top, SCREEN_NEAR, SCREEN_FAR) );
+		g_ProjectionStack.LoadMatrix( GetOrthoMatrix(left, right, bottom, top, SCREEN_NEAR, SCREEN_FAR) );
  		g_ModelViewStack.LoadIdentity();
 	}
 	else
@@ -310,7 +310,7 @@ void RageDisplay::LoadMenuPerspective(float fovDegrees)
 
 		/* It's the caller's responsibility to push first. */
 		g_ProjectionStack.LoadMatrix(
-			RageMatrixFrustrum(
+			GetFrustrumMatrix(
 			  -(SCREEN_WIDTH/2)/fDistCameraFromImage,
 			  +(SCREEN_WIDTH/2)/fDistCameraFromImage,
 			  +(SCREEN_HEIGHT/2)/fDistCameraFromImage,
@@ -350,7 +350,7 @@ void RageDisplay::EnterPerspective(float fov, bool preserve_loc, float near_clip
 	g_ModelViewStack.Push();
 
 	float aspect = SCREEN_WIDTH/(float)SCREEN_HEIGHT;
-	g_ProjectionStack.LoadMatrix( RageMatrixPerspective(fov, aspect, near_clip, far_clip) );
+	g_ProjectionStack.LoadMatrix( GetPerspectiveMatrix(fov, aspect, near_clip, far_clip) );
 	/* Flip the Y coordinate, so positive numbers go down. */
 	g_ProjectionStack.Scale(1, -1, 1);
 
@@ -400,9 +400,40 @@ void RageDisplay::ExitPerspective()
 	g_ModelViewStack.Pop();
 }
 
+
 /* gluLookAt.  The result is pre-multiplied to the matrix (M = L * M) instead of
  * post-multiplied. */
 void RageDisplay::LookAt(const RageVector3 &Eye, const RageVector3 &At, const RageVector3 &Up)
 {
 	PreMultMatrix(RageLookAt(Eye.x, Eye.y, Eye.z, At.x, At.y, At.z, Up.x, Up.y, Up.z));
 }
+
+RageMatrix RageDisplay::GetFrustrumMatrix( 
+	float left,    
+	float right,   
+	float bottom,  
+	float top,     
+	float znear,   
+	float zfar )	// see glFrustrum docs
+{
+	float A = (right+left) / (right-left);
+	float B = (top+bottom) / (top-bottom);
+	float C = -1 * (zfar+znear) / (zfar-znear);
+	float D = -1 * (2*zfar*znear) / (zfar-znear);
+	RageMatrix m(
+		2*znear/(right-left), 0,                   0,  0,
+		0,                   2*znear/(top-bottom), 0,  0,
+		A,                   B,                    C,  -1,
+		0,                   0,                    D,  0 );
+	return m;
+}
+
+RageMatrix RageDisplay::GetPerspectiveMatrix(float fovy, float aspect, float zNear, float zFar)
+{
+   float ymax = zNear * tanf(fovy * PI / 360.0f);
+   float ymin = -ymax;
+   float xmin = ymin * aspect;
+   float xmax = ymax * aspect;
+
+   return GetFrustrumMatrix(xmin, xmax, ymin, ymax, zNear, zFar);
+}

stepmania/src/RageDisplay.h

@@ -181,6 +181,16 @@ public:
 	void ExitPerspective();
 	void LookAt(const RageVector3 &Eye, const RageVector3 &At, const RageVector3 &Up);
 
+	RageMatrix GetOrthoMatrix( float l, float r, float b, float t, float zn, float zf ); 
+	RageMatrix GetFrustrumMatrix(
+		float left,    
+		float right,   
+		float bottom,  
+		float top,     
+		float znear,   
+		float zfar );
+	RageMatrix GetPerspectiveMatrix(float fovy, float aspect, float zNear, float zFar);
+
 protected:
 	const RageMatrix* GetProjection();
 	const RageMatrix* GetModelViewTop();

stepmania/src/RageDisplay_D3D.cpp

@@ -745,7 +745,8 @@ bool RageDisplay::IsZBufferEnabled() const
 
 void RageDisplay::SetZBuffer( bool b )
 {
-	g_pd3dDevice->SetRenderState( D3DRS_ZENABLE,      b );
+	g_pd3dDevice->SetRenderState( D3DRS_ZENABLE,      b ? D3DZB_TRUE : D3DZB_FALSE );
+	g_pd3dDevice->SetRenderState( D3DRS_ZFUNC,		  D3DCMP_LESSEQUAL );
 	g_pd3dDevice->SetRenderState( D3DRS_ZWRITEENABLE, b );
 }
 void RageDisplay::ClearZBuffer()
@@ -905,3 +906,15 @@ void RageDisplay::SetAlphaTest( bool b )
 	g_pd3dDevice->SetRenderState( D3DRS_ALPHAREF,         0 );
 	g_pd3dDevice->SetRenderState( D3DRS_ALPHAFUNC, D3DCMP_GREATER );
 }
+
+
+RageMatrix RageDisplay::GetOrthoMatrix( float l, float r, float b, float t, float zn, float zf )
+{
+	// D3DXMatrixOrthoOffCenterRH
+	RageMatrix m(
+		2/(r-l),      0,            0,           0,
+		0,            2/(t-b),      0,           0,
+		0,            0,            1/(zn-zf),   0,
+		(l+r)/(l-r),  (t+b)/(b-t),  zn/(zn-zf),  1 );
+	return m;
+}

stepmania/src/RageDisplay_OGL.cpp

@@ -999,3 +999,13 @@ void RageDisplay::SetAlphaTest( bool b )
 	else
 		glDisable( GL_ALPHA_TEST );
 }
+
+RageMatrix RageDisplay::GetOrthoMatrix( float l, float r, float b, float t, float zn, float zf )
+{
+	RageMatrix m(
+		2/(r-l),      0,            0,           0,
+		0,            2/(t-b),      0,           0,
+		0,            0,            -2/(zf-zn),   0,
+		-(r+l)/(r-l), -(t+b)/(t-b), -(zf+zn)/(zf-zn),  1 );
+	return m;
+}

stepmania/src/RageMath.cpp

@@ -440,23 +440,6 @@ RageMatrix RageLookAt(
 	return ret;
 }
 
-RageMatrix RageOrtho( float l, float r, float b, float t, float zn, float zf )
-{
-	RageMatrix m(
-		2/(r-l),      0,            0,           0,
-		0,            2/(t-b),      0,           0,
-		0,            0,            -2/(zf-zn),   0,
-		-(r+l)/(r-l), -(t+b)/(t-b), -(zf+zn)/(zf-zn),  1 );
-/*
-	RageMatrix m(
-		2/(r-l),      0,            0,           0,
-		0,            2/(t-b),      0,           0,
-		0,            0,            1/(zf-zn),   0,
-		(l+r)/(l-r),  (t+b)/(b-t),  zn/(zn-zf),  1 );
-*/
-	return m;
-}
-
 RageMatrix RageMatrixIdentity()
 {
 	RageMatrix m;
@@ -464,32 +447,3 @@ RageMatrix RageMatrixIdentity()
 	return m;
 }
 
-RageMatrix RageMatrixFrustrum( 
-	float left,    
-	float right,   
-	float bottom,  
-	float top,     
-	float znear,   
-	float zfar )	// see glFrustrum docs
-{
-	float A = (right+left) / (right-left);
-	float B = (top+bottom) / (top-bottom);
-	float C = -1 * (zfar+znear) / (zfar-znear);
-	float D = -1 * (2*zfar*znear) / (zfar-znear);
-	RageMatrix m(
-		2*znear/(right-left), 0,                   0,  0,
-		0,                   2*znear/(top-bottom), 0,  0,
-		A,                   B,                    C,  -1,
-		0,                   0,                    D,  0 );
-	return m;
-}
-
-RageMatrix RageMatrixPerspective(float fovy, float aspect, float zNear, float zFar)
-{
-   float ymax = zNear * tanf(fovy * PI / 360.0f);
-   float ymin = -ymax;
-   float xmin = ymin * aspect;
-   float xmax = ymax * aspect;
-
-   return RageMatrixFrustrum(xmin, xmax, ymin, ymax, zNear, zFar);
-}

stepmania/src/RageMath.h

@@ -41,14 +41,5 @@ RageMatrix RageLookAt(
 	float eyex, float eyey, float eyez,
 	float centerx, float centery, float centerz,
 	float upx, float upy, float upz );
-RageMatrix RageOrtho( float l, float r, float b, float t, float zn, float zf ); 
-RageMatrix RageMatrixFrustrum(
-	float left,    
-	float right,   
-	float bottom,  
-	float top,     
-	float znear,   
-	float zfar );
-RageMatrix RageMatrixPerspective(float fovy, float aspect, float zNear, float zFar);
 
 #endif

stepmania/src/ScreenSandbox.cpp

@@ -24,22 +24,29 @@
 
 ScreenSandbox::ScreenSandbox() : Screen("ScreenSandbox")
 {
-	m_quad.StretchTo( RectI(SCREEN_LEFT, SCREEN_TOP, SCREEN_RIGHT, SCREEN_BOTTOM) );
+	m_quad1.StretchTo( RectI(SCREEN_LEFT, SCREEN_TOP, SCREEN_RIGHT, SCREEN_BOTTOM) );
-	m_quad.SetDiffuse( RageColor(0,0,1,1) );
+	m_quad1.SetDiffuse( RageColor(0,0,1,1) );
-	m_quad.SetZ( -100 );
+	m_quad1.SetZ( 0 );
-	this->AddChild( &m_quad );
+	m_quad1.SetUseZBuffer( true );
+	this->AddChild( &m_quad1 );
+
+	m_quad2.StretchTo( RectI(SCREEN_LEFT, SCREEN_TOP, SCREEN_RIGHT, SCREEN_BOTTOM) );
+	m_quad2.SetDiffuse( RageColor(0,1,0,1) );
+	m_quad2.SetZ( 1 );
+	m_quad2.SetUseZBuffer( true );
+	this->AddChild( &m_quad2 );
 
 //	m_sprite.Load( "C:\\stepmania\\stepmania\\RandomMovies\\cm301[1].avi" );
 //	m_sprite.SetXY( CENTER_X, CENTER_Y );
 //	this->AddChild( &m_sprite );
 
-	m_model.SetXY(CENTER_X, CENTER_Y);
+//	m_model.SetXY(CENTER_X, CENTER_Y);
-	m_model.SetZoom(3);
+//	m_model.SetZoom(3);
-	//m_model.SetZoomY(-1);
+//	//m_model.SetZoomY(-1);
-	m_model.LoadMilkshapeAscii( "Down Tap Note 4th.txt" );
+//	m_model.LoadMilkshapeAscii( "Down Tap Note 4th.txt" );
 //	m_model.LoadMilkshapeAsciiBones( "D:\\Dev\\ddrpc char hacking\\DDRPCRip\\models\\howtoplay.bones.txt" );
-	this->AddChild(&m_model);
+//	this->AddChild(&m_model);
-	m_model.SetEffectSpin( RageVector3(0,90,90) );
+//	m_model.SetEffectSpin( RageVector3(0,90,90) );
 
 	this->AddChild( &m_In );
 
@@ -62,18 +69,18 @@ void ScreenSandbox::Update( float fDeltaTime )
 
 void ScreenSandbox::DrawPrimitives()
 {
-	DISPLAY->SetLighting( true );
+//	DISPLAY->SetLighting( true );
-	DISPLAY->SetLightDirectional( 
+//	DISPLAY->SetLightDirectional( 
-		0, 
+//		0, 
-		RageColor(0,0,0,0), 
+//		RageColor(0,0,0,0), 
-		RageColor(1,1,1,1),
+//		RageColor(1,1,1,1),
-		RageColor(0,0,0,1),
+//		RageColor(0,0,0,1),
-		RageVector3(0, -1, 0) );
+//		RageVector3(0, -1, 0) );
 
 	Screen::DrawPrimitives();
 
-	DISPLAY->SetLightOff( 0 );
+//	DISPLAY->SetLightOff( 0 );
-	DISPLAY->SetLighting( false );
+//	DISPLAY->SetLighting( false );
 }
 
 void ScreenSandbox::MenuLeft( PlayerNumber pn )

stepmania/src/ScreenSandbox.h

@@ -29,8 +29,9 @@ public:
 	void MenuLeft( PlayerNumber pn );
 	void MenuRight( PlayerNumber pn );
 
-	Model m_model;
+///	Model m_model;
-	Quad m_quad;
+	Quad m_quad1;
+	Quad m_quad2;
 	Transition	m_In;
 	Transition	m_Out;
 };