Update Cel shading. Note, RageDisplay_D3D will need to be fixed for this.

Data/Shaders/GLSL/Cel.frag

@@ -1,24 +1,26 @@
-varying vec2 texCoords;
+uniform sampler2D Texture0;
-varying vec3 normal, viewVector;
-varying vec4 lightVector, lightColor;
 
-uniform sampler2D Texture1;
+varying vec2 vCoord;
+varying vec3 vNor;
 
-void main(void)
+const vec4 shadowColor = vec4(0.6, 0.75, 0.9, 1.0);
-{
-	float intensity = dot(normal, lightVector.xyz);
-	float fresnel = pow(1.0 - dot(viewVector, normal), 5.0);
-	fresnel = fresnel < 0.5 ? 0.0 : 0.4;
 
-	float shade = 1.0;
+void main() {
-	if( intensity > 0.35 )
+	vec3 nor, light;
-		shade = clamp(1.0 * intensity + 0.4, 0.0, 1.0);
+	nor = normalize(vNor);
-	else
-		shade = clamp(0.85 * intensity + 0.25, 0.4, 0.6);
-	shade = clamp(shade - fresnel, 0.3, 1.0);
 	
-	vec4 vcol = gl_FrontMaterial.diffuse;
+	vec4 diffuse, specular, color, lightSource;
-	vec4 tex = texture2D(Texture1, texCoords);
+	float ambient = length(gl_FrontMaterial.ambient.rgb);
+	lightSource = gl_LightSource[0].position;
+	light = normalize(gl_ModelViewMatrix * lightSource).xyz;
 	
-	gl_FragColor = lightColor * vec4(tex.rgb * shade, 1.0);
+	color = texture2D(Texture0, vCoord.st);
+	float intensity = max(dot(light,nor), 0.0);
+	if (intensity < 0.5) {
+		intensity *= 0.5;
+		color *= shadowColor;
+	}
+	intensity = min(clamp(intensity, ambient, 1.0) + 0.25, 1.0);
+	
+	gl_FragColor = color * intensity;
 }

Data/Shaders/GLSL/Cel.vert

@@ -1,16 +1,9 @@
-varying vec3 normal, viewVector;
+varying vec2 vCoord;
-varying vec4 lightVector, lightColor;
+varying vec3 vNor;
-varying vec2 texCoords;
 
-void main(void)
+void main() {
-{
+	vCoord = gl_MultiTexCoord0.st;
-	texCoords = gl_MultiTexCoord0.st;
+	vNor = gl_Normal.xyz;
 
-	vec4 objectPos = gl_ModelViewMatrix * gl_Vertex;
+	gl_Position = gl_ModelViewProjectionMatrix * gl_Vertex;
-
-	normal = gl_NormalMatrix * gl_Normal;
-	lightVector = normalize(gl_LightSource[0].position);
-	lightColor = gl_LightSource[0].diffuse;
-	viewVector  = normalize(vec3(0) - objectPos.xyz);
-	gl_Position = ftransform();
 }

src/Model.cpp

@@ -289,18 +289,19 @@ bool Model::EarlyAbortDraw() const
 
 void Model::DrawCelShaded()
 {
-	// TODO: use shell shader for outline.
+	// First pass: shell. We only want the backfaces for this.
-	this->SetZWrite( true );
+	DISPLAY->SetCelShaded(1);
-
+	DISPLAY->SetCullMode(CULL_FRONT);
-	// First pass: outline/shell
+	this->SetZWrite(false); // XXX: Why on earth isn't the culling working? -Colby
-//	DISPLAY->SetCelShaded(1);
 	this->Draw();
 	
-	// Second pass: normal shading
+	// Second pass: cel shading
-//	DISPLAY->SetCelShaded(2);
+	DISPLAY->SetCelShaded(2);
-//	this->Draw();
+	DISPLAY->SetCullMode(CULL_BACK);
+	this->SetZWrite(true);
+	this->Draw();
 	
-//	DISPLAY->SetCelShaded(0)
+	DISPLAY->SetCelShaded(0);
 }
 
 void Model::DrawPrimitives()

src/RageDisplay.cpp

@@ -810,7 +810,7 @@ void RageDisplay::DrawQuads( const RageSpriteVertex v[], int iNumVerts )
 {
 	ASSERT( (iNumVerts%4) == 0 );
 
-	if(iNumVerts == 0)
+	if(!iNumVerts)
 		return;
 
 	this->DrawQuadsInternal(v,iNumVerts);

src/RageDisplay.h

@@ -277,7 +277,7 @@ public:
 		const RageVector3 &dir ) = 0;
 
 	virtual void SetSphereEnvironmentMapping( TextureUnit tu, bool b ) = 0;
-	virtual void SetCelShaded( bool b ) = 0;
+	virtual void SetCelShaded( int stage ) = 0;
 
 	virtual RageCompiledGeometry* CreateCompiledGeometry() = 0;
 	virtual void DeleteCompiledGeometry( RageCompiledGeometry* p ) = 0;

src/RageDisplay_Null.h

@@ -61,7 +61,7 @@ public:
 		const RageVector3 &dir ) { }
 
 	void SetSphereEnvironmentMapping( TextureUnit tu, bool b ) { }
-	void SetCelShaded( bool b ) { }
+	void SetCelShaded( int stage ) { }
 
 	RageCompiledGeometry* CreateCompiledGeometry();
 	void DeleteCompiledGeometry( RageCompiledGeometry* p );

src/RageDisplay_OGL.cpp

@@ -332,16 +332,18 @@ GLhandleARB CompileShader( GLenum ShaderType, RString sFile, vector<RString> asD
 
 GLhandleARB LoadShader( GLenum ShaderType, RString sFile, vector<RString> asDefines )
 {
+	// Don't do anything here if not the hardware/driver can't do it!
 	if( !GLExt.m_bGL_ARB_fragment_shader && ShaderType == GL_FRAGMENT_SHADER_ARB )
 		return 0;
 	if( !GLExt.m_bGL_ARB_vertex_shader && ShaderType == GL_VERTEX_SHADER_ARB )
 		return 0;
 
+	// XXX: dumb, but I don't feel like refactoring ragedisplay for this. -Colby
 	GLhandleARB secondaryShader = 0;
-	if( sFile == "Data/Shaders/GLSL/Cel.vert" )
+	if ( sFile == "Data/Shaders/GLSL/Cel.vert" )
-	{
 		secondaryShader = CompileShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Cel.frag", asDefines);
-	}
+	else if ( sFile == "Data/Shaders/GLSL/Shell.vert" )
+		secondaryShader = CompileShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Shell.frag", asDefines);
 	
 	GLhandleARB hShader = CompileShader( ShaderType, sFile, asDefines );
 	if( hShader == 0 )
@@ -349,7 +351,8 @@ GLhandleARB LoadShader( GLenum ShaderType, RString sFile, vector<RString> asDefi
 
 	GLhandleARB hProgram = GLExt.glCreateProgramObjectARB();
 	GLExt.glAttachObjectARB( hProgram, hShader );
-	if( secondaryShader != 0 )
+	
+	if( secondaryShader )
 	{
 		GLExt.glAttachObjectARB( hProgram, secondaryShader );
 		GLExt.glDeleteObjectARB( secondaryShader );
@@ -380,6 +383,7 @@ static GLhandleARB g_hHardMixShader = 0;
 static GLhandleARB g_hOverlayShader = 0;
 static GLhandleARB g_hScreenShader = 0;
 static GLhandleARB g_hYUYV422Shader = 0;
+static GLhandleARB g_gShellShader = 0;
 static GLhandleARB g_gCelShader = 0;
 
 void InitShaders()
@@ -388,16 +392,23 @@ void InitShaders()
 	// the shaders and determines shader type by file extension. -aj
 	// argh shaders in stepmania are painful -colby
 	vector<RString> asDefines;
-	g_bTextureMatrixShader = LoadShader( GL_VERTEX_SHADER_ARB, "Data/Shaders/GLSL/Texture matrix scaling.vert", asDefines );
+	
-	g_bUnpremultiplyShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Unpremultiply.frag", asDefines );
+	// used for scrolling textures (I think)
-	g_bColorBurnShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Color burn.frag", asDefines );
+	g_bTextureMatrixShader = LoadShader(	GL_VERTEX_SHADER_ARB, "Data/Shaders/GLSL/Texture matrix scaling.vert", asDefines );
-	g_bColorDodgeShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Color dodge.frag", asDefines );
+	
-	g_bVividLightShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Vivid light.frag", asDefines );
+	// these two are for dancing characters and are both actually shader pairs
-	g_hHardMixShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Hard mix.frag", asDefines );
+	g_gShellShader = LoadShader(			GL_VERTEX_SHADER_ARB, "Data/Shaders/GLSL/Shell.vert", asDefines );
-	g_hOverlayShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Overlay.frag", asDefines );
+	g_gCelShader = LoadShader(			GL_VERTEX_SHADER_ARB, "Data/Shaders/GLSL/Cel.vert", asDefines );
-	g_hScreenShader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Screen.frag", asDefines );
+	
-	g_hYUYV422Shader = LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/YUYV422.frag", asDefines );
+	// effects
-	g_gCelShader = LoadShader( GL_VERTEX_SHADER_ARB, "Data/Shaders/GLSL/Cel.vert", asDefines);
+	g_bUnpremultiplyShader	= LoadShader(	GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Unpremultiply.frag", asDefines );
+	g_bColorBurnShader	= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Color burn.frag", asDefines );
+	g_bColorDodgeShader	= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Color dodge.frag", asDefines );
+	g_bVividLightShader		= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Vivid light.frag", asDefines );
+	g_hHardMixShader		= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Hard mix.frag", asDefines );
+	g_hOverlayShader		= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Overlay.frag", asDefines );
+	g_hScreenShader		= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/Screen.frag", asDefines );
+	g_hYUYV422Shader		= LoadShader( GL_FRAGMENT_SHADER_ARB, "Data/Shaders/GLSL/YUYV422.frag", asDefines );
 	
 	// Bind attributes.
 	if( g_bTextureMatrixShader )
@@ -1699,8 +1710,8 @@ bool RageDisplay_OGL::IsEffectModeSupported( EffectMode effect )
 	case EffectMode_ColorDodge:	return g_bColorDodgeShader != 0;
 	case EffectMode_VividLight:	return g_bVividLightShader != 0;
 	case EffectMode_HardMix:	return g_hHardMixShader != 0;
-	case EffectMode_Overlay:	return g_hOverlayShader != 0;
+	case EffectMode_Overlay:		return g_hOverlayShader != 0;
-	case EffectMode_Screen:	return g_hScreenShader != 0;
+	case EffectMode_Screen:		return g_hScreenShader != 0;
 	case EffectMode_YUYV422:	return g_hYUYV422Shader != 0;
 	}
 
@@ -1902,14 +1913,14 @@ void RageDisplay_OGL::SetLightDirectional(
 
 void RageDisplay_OGL::SetCullMode( CullMode mode )
 {
+	if (mode != CULL_NONE)
+		glEnable(GL_CULL_FACE);
 	switch( mode )
 	{
 	case CULL_BACK:
-		glEnable( GL_CULL_FACE );
 		glCullFace( GL_BACK );
 		break;
 	case CULL_FRONT:
-		glEnable( GL_CULL_FACE );
 		glCullFace( GL_FRONT );
 		break;
 	case CULL_NONE:
@@ -2633,29 +2644,23 @@ void RageDisplay_OGL::SetSphereEnvironmentMapping( TextureUnit tu, bool b )
 
 GLint iCelTexture1, iCelTexture2 = NULL;
 
-void RageDisplay_OGL::SetCelShaded( bool b )
+void RageDisplay_OGL::SetCelShaded( int stage )
 {
-	if( GLExt.glUseProgramObjectARB == NULL )
+	if( !GLExt.m_bGL_ARB_fragment_shader )
 		return; // not supported
 
-	GLhandleARB hShader = 0;
+	switch ( stage )
-	if( b )
-		GLExt.glUseProgramObjectARB( g_gCelShader );
-	else
-		GLExt.glUseProgramObjectARB( hShader );
-
-	if( !b )
-		return;
-
-	/*
-	 * Optimization: don't get these again if we have already done it.
-	 * Getting data from the GPU is (relatively) slow, avoid it if possible.
-	 */
-	if( !iCelTexture1 )
 	{
-		iCelTexture1 = GLExt.glGetUniformLocationARB( hShader, "Texture1" );
+	case 1:
+		GLExt.glUseProgramObjectARB( g_gShellShader );
+		break;
+	case 2:
+		GLExt.glUseProgramObjectARB( g_gCelShader );
+		break;
+	default:
+		GLExt.glUseProgramObjectARB( 0 );
+		break;
 	}
-	GLExt.glUniform1iARB( iCelTexture1, 1 );
 }
 
 /*

src/RageDisplay_OGL.h

@@ -97,7 +97,7 @@ public:
 		const RageVector3 &dir );
 
 	void SetSphereEnvironmentMapping( TextureUnit tu, bool b );
-	void SetCelShaded( bool b );
+	void SetCelShaded( int stage );
 
 	RageCompiledGeometry* CreateCompiledGeometry();
 	void DeleteCompiledGeometry( RageCompiledGeometry* p );