In the version of WCF that shipped with .Net Framework 3.0 we didn't support custom validators with transport level HTTP security. We received much feedback from the community that this was a highly desired feature, so I'm happy to say we added support for this scenario in the 3.5 release of the .Net Framework.

Note that this is only supported under self hosted services.

Here is what you'll need to do to get it to work:

1. Create your validator:

namespace Sample
{
   using System.ServiceModel;
   using System.IdentityModel.Selectors;
   using System.ServiceModel.Security;

   public class CustomUserNameValidator : UserNamePasswordValidator
   {
       // This method validates users. It allows in two users, test1 and test2 
       // with passwords 1tset and 2tset respectively.
       // This code is for illustration purposes only and 
       // must not be used in a production environment because it is not secure. 
       public override void Validate(string userName, string password)
       {
           if (null == userName || null == password)
           {         
               throw new ArgumentNullException();
           }

           if (!(userName == "test1" && password == "1tset") && !(userName == "test2" && password == "2tset"))
           {
               throw new SecurityTokenException("Unknown Username or Incorrect Password");
           }
      }
   }
}

 2. Configure your service; you'll want to use Transport security and Basic clientCredentialType, this way the authentication header will be protected by SSL.

Configuration for the binding and behavior:

<bindings>
 <wsHttpBinding>
  <binding name="BasicAuthentication">
   <security mode="Transport">
    <transport clientCredentialType="Basic" />
   </security>
  </binding>
 </wsHttpBinding>
</bindings>


<behaviors>
 <serviceBehaviors>
  <behavior name="CustomUserNamePassword">
   <serviceCredentials>
    <userNameAuthentication 
     userNamePasswordValidationMode="Custom" 
     customUserNamePasswordValidatorType="Sample.CustomUserNameValidator, assemblyName"/>
   </serviceCredentials>
  </behavior>
 </serviceBehaviors>
</behaviors>

You can also add the custom validator through code:

host.Credentials.UserNameAuthentication.CustomUserNamePasswordValidator = new CustomUserNameValidator();
host.Credentials.UserNameAuthentication.UserNamePasswordValidationMode = UserNamePasswordValidationMode.Custom;