I've been stuck on this for a week, so I'm putting it out as a Friday puzzle challenge. Can you tell me what I'm doing wrong?

I added the normal mapping code from Followup: Normal Mapping Without Precomputed Tangents. I have other normal mapping code which uses the usual method of passing the tangent vector with the vertex data and that works great. There's no vertex shader code on The Tenth Planet, so I had to figure that out myself.

The problem I have is that if I use the normal from perturb_normal, the lighting rotates with the model so it is always the same side of the model that is lit (beautifully) no matter how the model is oriented relative to the light source. If I light the model with the interpolated vertex normal I get smooth n dot l lighting which works as expected.

To help with my debugging, I generated this completely flat normal map in Gimp. When using this it should act as though there is no normal map and light the model with smooth n dot l lighting using just the interpolated face normals.

The normal map is "blue-up" which seems to be the way the rock model normal maps are.



I'm not 100% sure that blue-up normal maps is what the code is expecting (given the discussion of green-up normals in the code) but this implementation's normal maps look like these so I don't think that accounts for what I'm seeing.

The code relevent to the problem is highlighted.

Vertex Shader:

uniform mat4 modelMatrix;
uniform mat4 viewMatrix;
uniform mat3 normalMatrix;
uniform mat4 modelViewMatrix;
uniform mat4 modelViewProjectionMatrix;
uniform mat4 shadowMatrix;
uniform vec3 lightPosition;		// World space light position
uniform vec3 cameraPosition;		// World space camera position
uniform vec4 fogColor;
uniform float fogDensity;

attribute vec3 vertex3;
attribute vec3 normal;
attribute vec2 texCoords0;

varying vec4 shadowCoord;
varying vec3 varyingNormal;
varying vec3 lightDirection;
varying vec3 viewDirection; 
varying vec2 texCoord0;
varying float fogFactor;

void main()
{
    vec4 vertex = vec4( vertex3.xyz, 1.0 );
    shadowCoord= shadowMatrix * modelMatrix * vertex;

vec4 vertexWorld = modelMatrix * vertex;
varyingNormal = ( modelMatrix * vec4( normal, 0.0 ) ).xyz;
viewDirection = cameraPosition - vertexWorld.xyz;
lightDirection = lightPosition - vertexWorld.xyz;

    vec4 vertexView = modelViewMatrix * vertex;
    const float LOG2 = 1.442695;
    float fogFragCoord = length( vertexView.xyz );
    fogFactor = exp2( -fogDensity * 
                       fogDensity * 
                       fogFragCoord * 
                       fogFragCoord * 
                       LOG2 );
    fogFactor = clamp(fogFactor, 0.0, 1.0);

    texCoord0 = texCoords0;
    gl_Position = modelViewProjectionMatrix * vertex;
}

Pixel Shader:

uniform sampler2D texture;
uniform sampler2D shadowMap;
uniform sampler2D normalMap;
uniform vec4 diffuseColor;      /* Color of diffuse light (the sun) */
uniform vec4 ambientColor;      /* Ambient light level, 0.1 0.1 0.1 1.0 */
uniform float shinyness;        /* 128.0 is very shiny (smaller bright spot), 1.0 for flat shading. */
uniform vec4 fogColor;

varying vec4 shadowCoord;
varying vec3 varyingNormal;
varying vec3 lightDirection;
varying vec3 viewDirection;
varying vec2 texCoord0;
varying float fogFactor;

vec4 normalizedShadowCoord;

float chebyshevUpperBound( float distance)
{
    /* Coordinates outside shadowMap, make them fully lit. */
    if ( ( normalizedShadowCoord.x <= 0.001 ) || ( 0.999 <= normalizedShadowCoord.x ) || 
         ( normalizedShadowCoord.y <= 0.001 ) || ( 0.999 <= normalizedShadowCoord.y ) )
        return 1.0;

    vec2 moments = texture2D( shadowMap, normalizedShadowCoord.xy ).rg;

    /* Area outside rendered section of shadow map is all white, detect and return fully lit. */
    if ( ( moments.r > 0.99999 ) && ( moments.g > 0.99999 ) ) 
        return 1.0;

    /* Surface is fully lit. as the current fragment is before the light occluder */
    if ( distance <= moments.x )
        return 1.0;

    /* The fragment is either in shadow or penumbra. We now use chebyshev's upperBound to check */
    /* How likely this pixel is to be lit (p_max) */
    float variance = moments.y - ( moments.x * moments.x );
    variance = max( variance, 0.0002 );

    float d = distance - moments.x;
    float p_max = variance / ( variance + d * d );

    /* Coordinates near edge of shadowmap, transition to fully lit. */
    float blendToWhite = 0.0;
    if ( normalizedShadowCoord.x <= 0.02 ) 
    {
        blendToWhite = ( 0.02 - normalizedShadowCoord.x ) / 0.02;
    }
    else if ( 0.98 <= normalizedShadowCoord.x ) 
    {
        blendToWhite = 1.0 - ( ( 1.0 - normalizedShadowCoord.x ) / 0.02 );
    }
    else if ( normalizedShadowCoord.y <= 0.02 ) 
    {
        blendToWhite = ( 0.02 - normalizedShadowCoord.y ) / 0.02;
    }
    else if ( 0.98 <= normalizedShadowCoord.y )
    {
        blendToWhite = 1.0 - ( ( 1.0 - normalizedShadowCoord.y ) / 0.02 );
    }

    return max( p_max, blendToWhite );
}

mat3 cotangent_frame( vec3 N, vec3 p, vec2 uv )
{
    /* get edge vectors of the pixel triangle */
    vec3 dp1 = dFdx( p );
    vec3 dp2 = dFdy( p );
    vec2 duv1 = dFdx( uv );
    vec2 duv2 = dFdy( uv );

    /* solve the linear system */
    vec3 dp2perp = cross( dp2, N );
    vec3 dp1perp = cross( N, dp1 );
    vec3 T = dp2perp * duv1.x + dp1perp * duv2.x;
    vec3 B = dp2perp * duv1.y + dp1perp * duv2.y;

    /* construct a scale-invariant frame */
    float invmax = inversesqrt( max( dot(T,T), dot(B,B) ) );
    return mat3( T * invmax, B * invmax, N );
}

vec3 perturb_normal( vec3 N, vec3 V, vec2 texcoord )
{
    /* assume N, the interpolated vertex normal and V, the view vector (vertex to eye) */
    vec3 map = texture2D( normalMap, texcoord ).xyz;
// WITH_NORMALMAP_UNSIGNED             
     map = map * 255./127. - 128./127.;
// WITH_NORMALMAP_2CHANNEL             
//     map.z = sqrt( 1. - dot( map.xy, map.xy ) ); 
// WITH_NORMALMAP_GREEN_UP                         
//     map.y = -map.y;                             
    mat3 TBN = cotangent_frame( N, -V, texcoord );
    return normalize( TBN * map );
}

void main (void)
{
    normalizedShadowCoord = shadowCoord / shadowCoord.w;
    float shadow = chebyshevUpperBound( normalizedShadowCoord.z );
// If this is uncommented, lighting rotates with the model
//    vec3 faceNormal = perturb_normal( normalize( varyingNormal ), normalize( viewDirection ), texCoord0.st );
// If this is uncommented, lighting works as expected (but is not normal mapped.)
vec3 faceNormal = normalize( varyingNormal );
    float diffuse = max( 0.0, dot( faceNormal, normalize( lightDirection ) ) );
    vec4 color = ( ( diffuse * diffuseColor * shadow ) + ambientColor ) * texture2D( texture, texCoord0.st );
    if ( ( diffuse != 0.0 ) && ( shinyness > 0.000001 ) )
    {
        vec3 reflection = normalize( reflect( -normalize( lightDirection ), faceNormal ) );
        float specular = max( 0.0, dot( faceNormal, reflection ) );
        float specularReflection = pow( specular, shinyness );
        color.rgb += vec3( specularReflection, specularReflection, specularReflection );
    }
    gl_FragColor = mix( fogColor, color, fogFactor );
}