Wcf discovery high cpu consumption problem

I have encountered a critical problem regarding wcf discovery.
The following WCF code used to try to find services and announce about my services.

  1: using System;
  2: using System.Collections.Generic;
  3: using System.Linq;
  4: using System.ServiceModel;
  5: using System.ServiceModel.Discovery;
  6: using System.Text;
  7: using System.Threading.Tasks;
  8: 
  9: namespace WpfApplication1
 10: {
 11:     public class OperatorServicesAccessor
 12:     {
 13: 
 14:         private AnnouncementService mAnnouncementService;
 15:         private ServiceHost mAnnouncementServiceHost;
 16:         UdpAnnouncementEndpoint mUdpAnnouncementEndpoint;
 17: 
 18:         private static OperatorServicesAccessor mOperatorServicesAccessor = new OperatorServicesAccessor();
 19: 
 20:         public static OperatorServicesAccessor Instance()
 21:         {
 22:             return mOperatorServicesAccessor;
 23:         }
 24: 
 25:         private OperatorServicesAccessor()
 26:         {
 27: 
 28:         }
 29: 
 30:         public void Init()
 31:         {
 32: 
 33:             StartScanForeOperatorEntities();
 34: 
 35:             OpenAnnouncementService();
 36:         }
 37: 
 38:         /// <summary>
 39:         /// When the client recevied non aplication exception it close the channel 
 40:         /// and wait for the service to start itself 
 41:         /// </summary>
 42:         /// <param name="pFaultedChannel"></param>
 43:         /// <param name="pEx"></param>
 44:         public void ReportAboutFaultedClient(object pFaultedChannel, Exception pEx)
 45:         {
 46: 
 47:             if (pFaultedChannel == null)
 48:             {
 49:                 return;
 50:             }
 51: 
 52:             if (pEx is ProtocolException)
 53:             {
 54: 
 55:             }
 56: 
 57:             if (pEx is CommunicationException)
 58:             {
 59: 
 60:             }
 61: 
 62:         }
 63: 
 64:         void mDiscoveryClient_FindCompleted(object sender, FindCompletedEventArgs e)
 65:         {
 66:             try
 67:             {
 68: 
 69:                 switch (e.UserState.ToString())
 70:                 {
 71:                     case "IOperator":
 72: 
 73:                         foreach (EndpointDiscoveryMetadata nextEndpointDiscoveryMetadata in e.Result.Endpoints)
 74:                         {
 75:                             EndpointAddress theAddress = nextEndpointDiscoveryMetadata.Address;
 76: 
 77:                             return;
 78:                         }
 79:                         break;
 80:                 }
 81:             }
 82:             catch (Exception ee)
 83:             {
 84: 
 85:             }
 86:             finally
 87:             {
 88:                 ((DiscoveryClient)sender).Close();
 89:             }
 90: 
 91:         }
 92: 
 93:         private void StartScanForeOperatorEntities()
 94:         {
 95: 
 96:             UdpDiscoveryEndpoint mUdpDiscoveryEndpoint = new UdpDiscoveryEndpoint();
 97: 
 98:             DiscoveryClient mDiscoveryClient = new DiscoveryClient(mUdpDiscoveryEndpoint);
 99: 
100:             mDiscoveryClient.FindCompleted += new EventHandler<FindCompletedEventArgs>(mDiscoveryClient_FindCompleted);
101: 
102:             FindCriteria criteria = new FindCriteria(typeof(IOperator));
103: 
104:             criteria.Duration = TimeSpan.FromSeconds(5);
105: 
106:             mDiscoveryClient.FindAsync(criteria, "IOperator");
107: 
108:         }
109: 
110:         private void OpenAnnouncementService()
111:         {
112:             mAnnouncementService = new AnnouncementService();
113:             // Add event handlers
114: 
115:             mAnnouncementService.OnlineAnnouncementReceived +=
116:                 new EventHandler<AnnouncementEventArgs>(this.OnOnlineAnnouncement);
117:             mAnnouncementService.OfflineAnnouncementReceived +=
118:                 new EventHandler<AnnouncementEventArgs>(this.OnOfflineAnnouncement);
119: 
120: 
121:             // Create the service host with a singleton
122:             mAnnouncementServiceHost = new ServiceHost(mAnnouncementService);
123:             // Add the announcement endpoint
124:             mUdpAnnouncementEndpoint = new UdpAnnouncementEndpoint();
125: 
126:             mAnnouncementServiceHost.AddServiceEndpoint(mUdpAnnouncementEndpoint);
127:             // Open the host async
128:             mAnnouncementServiceHost.Open();
129:         }
130:         /// <summary>
131:         /// 
132:         /// Get a notification from a cell that its power up 
133:         /// </summary>
134:         /// <param name="sender"></param>
135:         /// <param name="e"></param>
136:         private void OnOnlineAnnouncement(object sender, AnnouncementEventArgs e)
137:         {
138: 
139:         }
140: 
141:         private void OnOfflineAnnouncement(object sender, AnnouncementEventArgs e)
142:         {
143: 
144:         }
145:     }
146: }
147: 

A standard code for the mission of searching for services and registering  for hello goodbye announcements  .
When I execute the code the CPU consumption of the application is climbing and stay high for a while.
I found out that the WCF infrastructure generate a lot of exceptions after executing the code .
 

The visual studio debugger doesn't break in any exception so I try to figure out the exceptions source using windbg.

Using WinDBG
loading  sos :
.loadby sos clr
configure to catache the exceptions :
sxe clr

Then allot of exceptions started to  arrive:
(1298.1df8): CLR exception - code e0434352 (first chance)
First chance exceptions are reported before any exception handling.
This exception may be expected and handled.
KERNELBASE!RaiseException+0x54:
7579b760 c9              leave

Investigate the exceptions:
sxe 0xe0434352

0:000> !PrintException
Exception object: 01e45768
Exception type:   System.ArgumentException
Message:          The 'result' argument must be of type Message.
InnerException:   <none>
StackTrace (generated):
    SP       IP       Function
    0020ECC8 519CD54D System_ServiceModel_ni!System.ServiceModel.Dispatcher.MessageOperationFormatter.SerializeReply(System.ServiceModel.Channels.MessageVersion, System.Object[], System.Object)+0x336c31
    0020ECE4 50D2907F System_ServiceModel_ni!System.ServiceModel.Dispatcher.DispatchOperationRuntime.SerializeOutputs(System.ServiceModel.Dispatcher.MessageRpc ByRef)+0x8f
    0020ED44 50D26C42 System_ServiceModel_ni!System.ServiceModel.Dispatcher.DispatchOperationRuntime.InvokeBegin(System.ServiceModel.Dispatcher.MessageRpc ByRef)+0x562

0:000> !CLRStack
OS Thread Id: 0x1df8 (0)
Child SP       IP Call Site
0020ec0c 7579b760 [HelperMethodFrame: 0020ec0c]
0020ec7c 50d26cbf *** WARNING: Unable to verify checksum for C:\Windows\assembly\NativeImages_v4.0.30319_32\System.ServiceModel\23288b62c0dc0d181c1ffc0614644b6d\System.ServiceModel.ni.dll
*** ERROR: Module load completed but symbols could not be loaded for C:\Windows\assembly\NativeImages_v4.0.30319_32\System.ServiceModel\23288b62c0dc0d181c1ffc0614644b6d\System.ServiceModel.ni.dll
System.ServiceModel.Dispatcher.DispatchOperationRuntime.InvokeBegin(System.ServiceModel.Dispatcher.MessageRpc ByRef)
0020eda8 50d26582 System.ServiceModel.Dispatcher.ImmutableDispatchRuntime.ProcessMessage5(System.ServiceModel.Dispatcher.MessageRpc ByRef)
0020ede0 50d261eb System.ServiceModel.Dispatcher.ImmutableDispatchRuntime.ProcessMessage41(System.ServiceModel.Dispatcher.MessageRpc ByRef)
0020edf4 50d25067 System.ServiceModel.Dispatcher.MessageRpc.Process(Boolean)
0020ee40 50d33dcd System.ServiceModel.Dispatcher.MessageRpc+Wrapper.Resume(Boolean ByRef)
0020ee60 51484d49 System.ServiceModel.Dispatcher.ThreadBehavior.ResumeProcessing(System.ServiceModel.Dispatcher.IResumeMessageRpc)
0020ee7c 51484c26 System.ServiceModel.Dispatcher.ThreadBehavior.SynchronizationContextStartCallback(System.Object)
0020ee84 646077ef *** WARNING: Unable to verify checksum for C:\Windows\assembly\NativeImages_v4.0.30319_32\WindowsBase\4692c27dd31276c56a76ec0da8d3e453\WindowsBase.ni.dll
*** ERROR: Module load completed but symbols could not be loaded for C:\Windows\assembly\NativeImages_v4.0.30319_32\WindowsBase\4692c27dd31276c56a76ec0da8d3e453\WindowsBase.ni.dll
System.Windows.Threading.ExceptionWrapper.InternalRealCall(System.Delegate, System.Object, Int32)
0020eea4 6460768a MS.Internal.Threading.ExceptionFilterHelper.TryCatchWhen(System.Object, System.Delegate, System.Object, Int32, System.Delegate)
0020eee8 645f2d5c System.Windows.Threading.DispatcherOperation.InvokeImpl()

The cause of the exceptions is the WCF infrastructure !!!!
Configure the application  or logging the messages :

  1: <?xml version="1.0" encoding="utf-8" ?>
  2: <configuration>
  3:     <startup> 
  4:         <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
  5:     </startup>
  6:   <system.serviceModel>
  7: 
  8: 
  9:     <diagnostics>
 10:       <!-- Enable Message Logging here -->
 11:       <!-- log all messages received or sent at the transport or service model levels -->
 12:       <messageLogging logEntireMessage="true" maxMessagesToLog="300" logMessagesAtServiceLevel="true" logMalformedMessages="true" logMessagesAtTransportLevel="true"/>
 13:     </diagnostics>
 14: 
 15:   </system.serviceModel>
 16: 
 17:   <system.diagnostics>
 18:     <sources>
 19:       <source name="System.ServiceModel" switchValue="Information,ActivityTracing" propagateActivity="true">
 20:         <listeners>
 21:           <add name="xml"/>
 22:         </listeners>
 23:       </source>
 24:       <source name="System.ServiceModel.MessageLogging">
 25:         <listeners>
 26:           <add name="xml"/>
 27:         </listeners>
 28:       </source>
 29:     </sources>
 30:     <sharedListeners>
 31:       <add initializeData="C:\logs\SimulatorGuiB10.svclog" type="System.Diagnostics.XmlWriterTraceListener" name="xml"/>
 32:     </sharedListeners>
 33:     <trace autoflush="true"/>
 34:   </system.diagnostics>
 35: </configuration>

Inspect the  log : The message with Action 'http://schemas.microsoft.com/net/2005/12/windowscommunicationfoundation/dispatcher/fault' cannot be processed at the receiver, due to a ContractFilter mismatch at the EndpointDispatcher. This may be because of either a contract mismatch (mismatched Actions between sender and receiver) or a binding/security mismatch between the sender and the receiver.  Check that sender and receiver have the same contract and the same binding (including security requirements, e.g. Message, Transport, None). 

I understand that there  is  contract mismatch

So I checked the address of the Discovery endpoint and the Announcement endpoint

The contract is different between them  but the address is the same .
Resolving the problem:
Set the address manually to be different solve the problem:

  1: private void OpenAnnouncementService()
  2:         {
  3:             mAnnouncementService = new AnnouncementService();
  4:             // Add event handlers
  5: 
  6:             mAnnouncementService.OnlineAnnouncementReceived +=
  7:                 new EventHandler<AnnouncementEventArgs>(this.OnOnlineAnnouncement);
  8:             mAnnouncementService.OfflineAnnouncementReceived +=
  9:                 new EventHandler<AnnouncementEventArgs>(this.OnOfflineAnnouncement);
 10: 
 11: 
 12:             // Create the service host with a singleton
 13:             mAnnouncementServiceHost = new ServiceHost(mAnnouncementService);
 14:             // Add the announcement endpoint
 15:             mUdpAnnouncementEndpoint = new UdpAnnouncementEndpoint();
 16: 
 17:             mUdpAnnouncementEndpoint.MulticastAddress = new Uri("soap.udp://239.255.255.251:3702/");
 18: 
 19:             mAnnouncementServiceHost.AddServiceEndpoint(mUdpAnnouncementEndpoint);
 20:             // Open the host async
 21:             mAnnouncementServiceHost.Open();
 22:         }

wcf calls thread affinity

A week ago we have notice that many WCF service calls are executing in the UI thread context.
We wish to eliminate this behavior in order to void blocking the UI thread by WCF service calls.
Because the WCF service calls execution time is very long we wants that every call will be execute in a new thread and not using a thread pool .
The solution for this problem was :

Create custom SynchronizationContext that overrides the send and post methods

  1: using System;
  2: using System.Collections.Generic;
  3: using System.Linq;
  4: using System.Text;
  5: using System.Threading;
  6: 
  7: namespace WcfInfra
  8: {
  9:     public class NewThreadForEachCallSyncContext: SynchronizationContext
 10:     {
 11:         public NewThreadForEachCallSyncContext ()
 12: 	    {
 13: 
 14: 	    }
 15:         public override void Send(SendOrPostCallback d, object state)
 16:         {
 17:             new Thread(new ThreadStart(() =>
 18:             {
 19:                 d(state);
 20:             })).Start();
 21:         }
 22:         public override void Post(SendOrPostCallback d, object state)
 23:         {
 24:             new Thread (new ThreadStart (()=>
 25:             {
 26:                 d(state);
 27:             })).Start ();
 28:         }
 29: 
 30:     }
 31: }

Create a wcf service attribute that implements the IContractBehavior .In the applydispatchbehavior attach to the dispatchRuntime.SynchronizationContext  an instance of our custom SynchronizationContext .

  1: public class NewThreadForEachCallInterceptorAttribute : Attribute, IContractBehavior
  2: {
  3: 	public NewThreadForEachCallInterceptorAttribute()
  4: 	{
  5: 
  6: 	}
  7: 
  8: 	public void AddBindingParameters(ContractDescription contractDescription, ServiceEndpoint endpoint, System.ServiceModel.Channels.BindingParameterCollection bindingParameters)
  9: 	{
 10: 		
 11: 	}
 12: 
 13: 	public void ApplyClientBehavior(ContractDescription contractDescription, ServiceEndpoint endpoint, System.ServiceModel.Dispatcher.ClientRuntime clientRuntime)
 14: 	{
 15: 		
 16: 	}
 17: 
 18: 	public void ApplyDispatchBehavior(ContractDescription contractDescription, ServiceEndpoint endpoint, System.ServiceModel.Dispatcher.DispatchRuntime dispatchRuntime)
 19: 	{
 20: 		dispatchRuntime.SynchronizationContext = new TrainerSyncContext();
 21: 	}
 22: 
 23: 	public void Validate(ContractDescription contractDescription, ServiceEndpoint endpoint)
 24: 	{
 25: 
 26: 	}
 27: }

Adore the wcf service with the new attribute

  1: [ServiceBehavior(IncludeExceptionDetailInFaults = true ,InstanceContextMode = InstanceContextMode.Single, ConcurrencyMode = ConcurrencyMode.Multiple)]
  2:     [NewThreadForEachCallInterceptor()]
  3:     [ErrorBehavior(typeof(ApplicationException))]
  4:     public class DoThingsLongTimeService : IDoThing

Discovery service name and namespace

Setting the name and the namespace of a WCF service for discovery announcement can be done using the ServiceContract attribute properties
For an example:
[ServiceContract(Namespace=@"Com.myCompany.Services",Name="IMyServiceAlias"

)]
public interface IMyService

The change can be seen using Wireshark

<s:Envelope xmlns:s="http://www.w3.org/2003/05/soap-envelope" xmlns:a="http://www.w3.org/2005/08/addressing">
  <s:Header>
    <a:Action s:mustUnderstand="1">http://docs.oasis-open.org/ws-dd/ns/discovery/2009/01/Hello</a:Action>
    <h:AppSequence InstanceId="1369239226" MessageNumber="1" xmlns:h="http://docs.oasis-open.org/ws-dd/ns/discovery/2009/01"/>
    <a:MessageID>urn:uuid:bc95d269-a20f-4d7b-b5bd-c31698dd9569</a:MessageID>
    <a:To s:mustUnderstand="1">urn:docs-oasis-open-org:ws-dd:ns:discovery:2009:01</a:To>
  </s:Header>
  <s:Body>
    <Hello xmlns="http://docs.oasis-open.org/ws-dd/ns/discovery/2009/01">
      <a:EndpointReference>
        <a:Address>http://localhost:8989/myService</a:Address>
      </a:EndpointReference>
      <d:Types xmlns:d="http://docs.oasis-open.org/ws-dd/ns/discovery/2009/01" xmlns:dp0="Com.myCompany.Services">dp0:IMyServiceAlias</d:Types>
      <XAddrs>http://localhost:8989/myService</XAddrs>
      <MetadataVersion>0</MetadataVersion>
    </Hello>
  </s:Body>
</s:Envelope>