Click here to Skip to main content
15,887,453 members
Please Sign up or sign in to vote.
0.00/5 (No votes)
I'm trying to apply a cel or toon effect to an image. I got the HLSL effect from http://rbwhitaker.wikidot.com/toon-shader[^].

The thing is, when I'm compiling the effect using SharpDX Toolkit I'm having two error types. I'm trying to compile with net40 SharpDXToolkit, use following instruction.

C:\Projects\SharpDXBin\Bin\DirectX11_1-net40>tkfxc.exe /ICToon.tkfxo ToonColore
d.fx


Errors are:
1. Converting float3 to float4. (type casting worked once)
2. X4502: invalid ps_4_0_level_9_1 output semantic 'COLOR0'

Here's the code.

C++
//--------------------------- BASIC PROPERTIES ------------------------------
// The world transformation
float4x4 World;
 
// The view transformation
float4x4 View;
 
// The projection transformation
float4x4 Projection;
 
// The transpose of the inverse of the world transformation,
// used for transforming the vertex's normal
float4x4 WorldInverseTranspose;
 
//--------------------------- DIFFUSE LIGHT PROPERTIES ------------------------------
// The direction of the diffuse light
float3 DiffuseLightDirection = float3(1, 0, 0);
 
// The color of the diffuse light
float4 DiffuseColor = float4(1, 1, 1, 1);
 
// The intensity of the diffuse light
float DiffuseIntensity = 1.0;
 
//--------------------------- TOON SHADER PROPERTIES ------------------------------
// The color to draw the lines in.  Black is a good default.
float4 LineColor = float4(0, 0, 0, 1);
 
// The thickness of the lines.  This may need to change, depending on the scale of
// the objects you are drawing.
float LineThickness = .03;
 
//--------------------------- DATA STRUCTURES ------------------------------
// The structure used to store information between the application and the
// vertex shader
struct AppToVertex
{
    float4 Position : POSITION0;            // The position of the vertex
    float3 Normal : NORMAL0;                // The vertex's normal
	float4 Color : COLOR0;                  // The vertex's color
};
 
// The structure used to store information between the vertex shader and the
// pixel shader
struct VertexToPixel
{
    float4 Position : POSITION0;
    float4 Color : COLOR0;
    float3 Normal : TEXCOORD1;
};
 
//--------------------------- SHADERS ------------------------------
// The vertex shader that does cel shading.
// It really only does the basic transformation of the vertex location,
// and normal, and copies the texture coordinate over.
VertexToPixel CelVertexShader(AppToVertex input)
{
    VertexToPixel output;
 
    // Transform the position
    float4 worldPosition = mul(input.Position, World);
    float4 viewPosition = mul(worldPosition, View);
    output.Position = mul(viewPosition, Projection);
 
    // Transform the normal
    output.Normal = normalize(mul(input.Normal, (float3x3)WorldInverseTranspose));
 
    // Copy over the texture coordinate
    output.Color = input.Color;
 
    return output;
}
 
// The pixel shader that does cel shading.  Basically, it calculates
// the color like is should, and then it discretizes the color into
// one of four colors.
float4 CelPixelShader(VertexToPixel input) : COLOR0
{
    // Calculate diffuse light amount
    float intensity = dot(normalize(DiffuseLightDirection), input.Normal);
    if(intensity < 0)
        intensity = 0;
 
    // Calculate what would normally be the final color, including texturing and diffuse lighting
    float4 color = input.Color * DiffuseColor * DiffuseIntensity;
    color.a = 1;
 
    // Discretize the intensity, based on a few cutoff points
    if (intensity > 0.95)
        color = float4(1.0,1,1,1.0) * color;
    else if (intensity > 0.5)
        color = float4(0.7,0.7,0.7,1.0) * color;
    else if (intensity > 0.05)
        color = float4(0.35,0.35,0.35,1.0) * color;
    else
        color = float4(0.1,0.1,0.1,1.0) * color;
 
    return color;
}
 
// The vertex shader that does the outlines
VertexToPixel OutlineVertexShader(AppToVertex input)
{
    VertexToPixel output = (VertexToPixel)0;
 
    // Calculate where the vertex ought to be.  This line is equivalent
    // to the transformations in the CelVertexShader.
    float4 original = mul(mul(mul(input.Position, World), View), Projection);
 
    // Calculates the normal of the vertex like it ought to be.
    float4 normal = mul(mul(normalize(mul(float4(input.Normal.xyz, 0),World)), View), Projection);
 
    // Take the correct "original" location and translate the vertex a little
    // bit in the direction of the normal to draw a slightly expanded object.
    // Later, we will draw over most of this with the right color, except the expanded
    // part, which will leave the outline that we want.
    output.Position    = original + (mul(LineThickness, normal));
 
    return output;
}
 
// The pixel shader for the outline.  It is pretty simple:  draw everything with the
// correct line color.
float4 OutlinePixelShader(VertexToPixel input) : COLOR0
{
    return LineColor;
}
 
// The entire technique for doing toon shading
technique Toon
{
    // The first pass will go through and draw the back-facing triangles with the outline shader,
    // which will draw a slightly larger version of the model with the outline color.  Later, the
    // model will get drawn normally, and draw over the top most of this, leaving only an outline.
    pass Pass1
    {
        VertexShader = compile vs_2_0 OutlineVertexShader();
        PixelShader = compile ps_2_0 OutlinePixelShader();
        CullMode = CW;
    }
 
    // The second pass will draw the model like normal, but with the cel pixel shader, which will
    // color the model with certain colors, giving us the cel/toon effect that we are looking for.
    pass Pass2
    {
        VertexShader = compile vs_2_0 CelVertexShader();
        PixelShader = compile ps_2_0 CelPixelShader();
        CullMode = CCW;
    }
}



I've changed these lines from the original one:

C++
output.Normal = normalize(mul(input.Normal, WorldInverseTranspose));

// Calculates the normal of the vertex like it ought to be.
    float4 normal = mul(mul(mul(input.Normal, World), View), Projection);


for these others, just to try a sucessfull compilation:

C++
output.Normal = normalize(mul(input.Normal, (float3x3)WorldInverseTranspose));

// Calculates the normal of the vertex like it ought to be.
    float4 normal = mul(mul(normalize(mul(float4(input.Normal.xyz, 0),World)), View), Projection);


Compiler throw these errors:

SharpDX Toolkit Effect Compiler - 2.5.0.0
Copyright © 2010-2013 Alexandre Mutel

Compile Effect File [C:\Projects\SharpDXBin\Bin\DirectX11_1-net40\ToonColored.f
x]
Error when compiling file [ToonColored.fx]:
error X4541: vertex shader must minimally write all four components of SV_Positi
on

C:\\Projects\\SharpDXBin\\Bin\\DirectX11_1-net40\\ToonColored.fx(124,51-58): er
ror X4502: invalid ps_4_0_level_9_1 output semantic 'COLOR0'

error X4541: vertex shader must minimally write all four components of SV_Positi
on

C:\\Projects\\SharpDXBin\\Bin\\DirectX11_1-net40\\ToonColored.fx(77,47-54): err
or X4502: invalid ps_4_0_level_9_1 output semantic 'COLOR0'

But now I'm stuck, any help would be appreciated.
Posted
Updated 24-Aug-13 2:56am
v4

1 solution

Ok, the solution is here http://msdn.microsoft.com/en-us/library/windows/desktop/bb509647(v=vs.85).aspx#VPOS[^]

The script was written for DirectX9 and I was trying to compile it in DirectX10, so I just need to change from POSITION to SV_POSITION and from COLOR0 (only in retorn function, not in structure definition) to SV_Target.

Now it's compiled, even I can rollback changes made to original one.
 
Share this answer
 
v2

This content, along with any associated source code and files, is licensed under The Code Project Open License (CPOL)



CodeProject, 20 Bay Street, 11th Floor Toronto, Ontario, Canada M5J 2N8 +1 (416) 849-8900