Decrypt my World

How to manipulate REG_MULTI_SZ values from the registry (C++)

2009-11-12T15:48:00Z

Hi all,

The other day I had to develop a small C++ sample which shows how to read the list of values of a REG_MULTI_SZ from the registry, and add a new value just after one of the values of the list. Additionally, I used methods and constants from tchar.h extensively, so it didn't matter if I compiled the code as UNICODE or ANSI. Here is the code:

#include "windows.h"
#include "tchar.h"
#include "conio.h"
#include "stdio.h"

#define MY_KEY _T("PathToMyRegistryKey\\MyRegistryKey") // Registry key
#define MY_VALUES _T("NameOfTheREG_MULTI_SZListOfValues") // Registry values
#define NEW_VALUE _T("MyNewValue") // New value
#define FIND_VALUE _T("AnExistingValue") // We will insert the new value after this one

int _tmain(int argc, _TCHAR* argv[])
{
	LONG lResult = 0;
	HKEY hKey = NULL;
	LPTSTR lpValues = NULL;
	LPTSTR lpValue = NULL;
	LPTSTR lpNewValues = NULL;
	LPTSTR lpNewValue = NULL;
	DWORD cbValues = 0;
	DWORD cbNewValues = 0;
	DWORD cbNewValue = 0;
	BOOL bFound = FALSE;

	__try 
	{
		// OPEN THE REGISTRY KEY
		//
		_tprintf(_T("RegOpenKeyEx..."));
		lResult = RegOpenKeyEx(
			HKEY_LOCAL_MACHINE, 
			MY_KEY, 
			0,
			KEY_ALL_ACCESS,
			&hKey
		);
		if (ERROR_SUCCESS != lResult) { _tprintf(_T("ERROR 0x%x\n"), lResult); return 1; } 
		_tprintf(_T("SUCCESS\n"));

		// READ THE REG_MULTI_SZ VALUES
		//
		// Get size of the buffer for the values
		_tprintf(_T("RegQueryValueEx..."));
		lResult = RegQueryValueEx(
			hKey,
			MY_VALUES,
			NULL,
			NULL,
			NULL,
			&cbValues
		);
		if (ERROR_SUCCESS != lResult) { _tprintf(_T("ERROR 0x%x\n"), lResult); return 1; } 
		_tprintf(_T("SUCCESS\n"));

		// Allocate the buffer
		_tprintf(_T("malloc..."));
		lpValues = (LPTSTR)malloc(cbValues);
		if (NULL == lpValues) { _tprintf(_T("ERROR 0x%x\n"), GetLastError()); return 1; } 
		_tprintf(_T("SUCCESS\n"));

		// Get the values
		_tprintf(_T("RegQueryValueEx..."));
		lResult = RegQueryValueEx(
			hKey,
			MY_VALUES, 
			NULL,
			NULL,
			(LPBYTE)lpValues,
			&cbValues
		);
		if (ERROR_SUCCESS != lResult) { _tprintf(_T("ERROR 0x%x\n"), lResult); return 1; } 
		_tprintf(_T("SUCCESS\n"));

		// SHOW THE VALUES
		//
		_tprintf(_T("\n**************************\n"));
		_tprintf(_T("OLD VALUES\n"));
		_tprintf(_T("**************************\n\n"));
		lpValue = lpValues;
		for (; '\0' != *lpValue; lpValue += _tcslen(lpValue) + 1)
		{
			// Show one value
			_tprintf(_T("%s\n"), lpValue);
		}
		_tprintf(_T("\n**************************\n\n"));

		// INSERT A NEW VALUE AFTER A SPECIFIC VALUE IN THE LIST OF VALUES
		//
		// Allocate a new buffer for the old values plus the new one
		_tprintf(_T("malloc..."));
		cbNewValue = (_tcslen(NEW_VALUE) + 1) * sizeof(TCHAR);
		cbNewValues = cbValues + cbNewValue;
		lpNewValues = (LPTSTR)malloc(cbNewValues);
		if (NULL == lpNewValues) { _tprintf(_T("ERROR 0x%x\n"), GetLastError()); return 1; } 
		_tprintf(_T("SUCCESS\n"));		

		// Find the value after which we will insert the new one
		lpValue = lpValues;
		lpNewValue = lpNewValues;
		bFound = FALSE;
		for (; '\0' != *lpValue; lpValue += _tcslen(lpValue) + 1)
		{
			// Copy the current value to the target buffer
			memcpy(lpNewValue, lpValue, (_tcslen(lpValue) + 1) * sizeof(TCHAR));

			if (0 == _tcscmp(lpValue, FIND_VALUE))
			{
				// The current value is the one we wanted to find
				bFound = TRUE;

				// Copy the new value to the target buffer
				lpNewValue += _tcslen(lpValue) + 1;
				memcpy(lpNewValue, NEW_VALUE, (_tcslen(NEW_VALUE) + 1) * sizeof(TCHAR));
				lpNewValue += _tcslen(NEW_VALUE) + 1;
			}
			else
			{
				// This is not the value we want, continue to the next one
				lpNewValue += _tcslen(lpValue) + 1;
			}
		}
		if (!bFound) 
		{ 
			// We didn't find the value we wanted. Insert the new value at the end
			memcpy(lpNewValue, NEW_VALUE, (_tcslen(NEW_VALUE) + 1) * sizeof(TCHAR));
			lpNewValue += _tcslen(NEW_VALUE) + 1;
		}
		*lpNewValue = *lpValue;

		// SHOW THE NEW VALUES
		//
		_tprintf(_T("\n**************************\n"));
		_tprintf(_T("NEW VALUES\n"));
		_tprintf(_T("**************************\n\n"));
		lpNewValue = lpNewValues;
		for (; '\0' != *lpNewValue; lpNewValue += _tcslen(lpNewValue) + 1)
		{
			// Show one value
			_tprintf(_T("%s\n"), lpNewValue);
		}
		_tprintf(_T("\n**************************\n\n"));

		// WRITE THE NEW VALUES BACK TO THE KEY
		//
		_tprintf(_T("RegSetValueEx..."));
		lResult = RegSetValueEx(
			hKey,
			MY_VALUES, 
			NULL,
			REG_MULTI_SZ,
			(LPBYTE)lpNewValues,
			cbNewValues
		);
		if (ERROR_SUCCESS != lResult) { _tprintf(_T("ERROR 0x%x\n"), lResult); return 1; } 
		_tprintf(_T("SUCCESS\n"));
	}
	__finally
	{
		// Clean up	
		//
		if (NULL != lpValues) { free(lpValues); }
		if (NULL != lpNewValues) { free(lpNewValues); }
		if (NULL != hKey) { RegCloseKey(hKey); }

		//_tprintf(_T("\n<<PRESS ANY KEY>>\n"));
		//_getch();
	}

	return 0;
}

Regards,

Alex (Alejandro Campos Magencio)

Access Violation exception when loading a certificate in .NET (Windows Server 2003)

2009-11-12T15:32:00Z

Hi all

You may get an Access Violation exception when loading a certificate into a X509Certificate2 object, only after installing KB 938397 hotfix (crypt32.dll v5.131.3790.4095) at Applications that use the Cryptography API cannot validate an X.509 certificate in Windows Server 2003.

Note that the issue won't happen with Windows Server 2003 SP2 version of crypt32.dll (v5.131.3790.3959).

Note that the issue only happens with Base64 encoded certificate files. DER encoded certificate files work just fine.

The good news is that a newer hotfix, KB 950474, installs a newer version of crypt32.dll that fixes the issue: v5.131.3790.4271. You can get it here: Error message when you run the SharePoint Products and Technologies Configuration Wizard on a Windows Server 2003-based computer: "Application has generated an exception that could not be handled".

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

PS: What are you still doing with Windows Server 2003? Have you tried the new Windows Server 2008 R2? IT JUST ROCKS!!!!

CAPICOM support on Windows 7

2009-10-15T18:55:00Z

Hi all,

CAPICOM has finally been deprecated, and it won't be supported on Windows 7. This link is up to date and proposes alternatives to CAPICOM classes by using .NET classes and CryptoAPI: Alternatives to Using CAPICOM.

The following article may help if you were using CAPICOM in your web site and you are planning on using .NET now: Writing an ActiveX Control in .NET.

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

Working with CNG in .NET 3.5

2009-10-14T10:37:00Z

Hi all,

Do you want to work with CNG and .NET 3.5 and don't want to P/Invoke into CryptoAPI & CNG API? Check out Security.Cryptography.dll library at CLR Security:

"
Security.Cryptography.dll provides a new set of algorithm implementations to augment the built in .NET framework supported algorithms. It also provides some APIs to extend the existing framework cryptography APIs. Within this project you will find:
§ A CNG implementation of the AES, RSA, and TripleDES encryption algorithms
§ A CNG implementation of a random number generator
§ A class that allows dynamically creating algorithms both from this library as well as all of the algorithms that ship with .NET 3.5
§ An enumerator over all of the installed CNG providers on the current machine
§ Extension methods that allow access to all of the keys installed in a CNG provider, as well as all of the algorithms the provider supports
"

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

CryptographicException: The data to be decrypted exceeds the maximum for this modulus of 0 bytes

2009-10-09T14:06:00Z

Hi all,

Some time ago a customer of mine was getting the following CryptographicException when calling EncryptedXml.DecryptDocument method to decrypt an XmlDocument in their .NET app:

[System.Security.Cryptography.CryptographicException] = {"The data to be decrypted exceeds the maximum for this modulus of 0 bytes"}
_HResult = -2146233296
_HResult = 0x80131430
No Inner Exception

This was only happening with a smart card certificate associated to a specific third-party CSP (non-MS).

In order to troubleshoot this, I used my CryptoAPI Tracer script to get some traces while reproducing the issue. Thanks to my script I could see these calls that .NET is making to the third-party CSP behind the scenes (I will omit key container and CSP names as they are not needed to understand the issue):

>>>>>>>>>>>>>>>>>>>>>>

CryptAcquireContextA (0x570)

IN
pszContainer
001bf1a0 "Cert Container Name"

pszProvider
001b4698 "Third-party CSP"

dwProvType
PROV_RSA_FULL

dwFlags
0

OUT
hProv
0x1bea28

RESULT
CryptAcquireContextA (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

>>>>>>>>>>>>>>>>>>>>>>

CryptGetUserKey (0x570)

IN
hProv
0x1bea28

dwKeySpec
AT_KEYEXCHANGE

OUT
hUserKey
0x1bf1a0

RESULT
CryptGetUserKey (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

>>>>>>>>>>>>>>>>>>>>>>

CryptGetKeyParam (0x570)

IN
hKey
0x1bf1a0

dwParam
KP_ALGID

pbData
NULL

dwDataLen
0

dwFlags
0

OUT
dwDataLen
4

RESULT
CryptGetKeyParam (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

>>>>>>>>>>>>>>>>>>>>>>

CryptGetKeyParam (0x570)

IN
hKey
0x1bf1a0

dwParam
KP_ALGID

pbData
0x1bd228

dwDataLen
4

dwFlags
0

OUT
bData
001bd228 0000a400

dwDataLen
4

RESULT
CryptGetKeyParam (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

>>>>>>>>>>>>>>>>>>>>>>

CryptGetKeyParam (0x570)

IN
hKey
0x1bf1a0

dwParam
KP_KEYLEN

pbData
NULL

dwDataLen
0

dwFlags
0

OUT
dwDataLen
4

RESULT
CryptGetKeyParam (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

>>>>>>>>>>>>>>>>>>>>>>

CryptGetKeyParam (0x570)

IN
hKey
0x1bf1a0

dwParam
KP_KEYLEN

pbData
0x1bd228

dwDataLen
4

dwFlags
0

OUT
bData
001bd228 00000000

dwDataLen
4

RESULT
CryptGetKeyParam (0x570) SUCCEEDED

<<<<<<<<<<<<<<<<<<<<<<

These calls mean the following: before .NET tries to decrypt the XML, it asks the CSP for some information on the key it will use to decrypt, like its length (CryptGetKeyParam API with KP_KEYLEN flag). For some unknown reason the third-party CSP is returning a length of 0 for the key (pbData points to 0), which is invalid, thus .NET won't continue with the decryption and will raise the exception we've seen. Why is the CSP returning an invalid length? Only the CSP developers can answer that question.

In my customer's particular case, the CSP developers quickly localized and fixed the issue.

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

X509Certificate creates temporary files which are never deleted

2009-09-30T11:52:00Z

Hi all,

When creating an X509Certificate object in your .NET application ("X509Certificate cert = new X509Certificate(certBytes);") you may see that two 0kb files are created in the Temp folder of your Windows Server 2003, and are never deleted. After creating many of those objects the Temp folder fills up and the app won't function properly.

The cause of this issue is the following bug:

On a Windows Server 2003-based client computer, the system does not delete a temporary file that is created when an application calls the "CryptQueryObject" function

X509Certificate class uses CryptQueryObject API behind the scenes. Fortunatelly there is a fix for this referenced on the link above.

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

How to timestamp Authenticode signatures when our proxy requires authentication

2009-09-29T13:47:00Z

Hi all,

The other day a customer of mine was trying to add an Authenticode signature with timestamp to their PowerShell scripts with PowerShell itself and its Set-AuthenticodeSignature cmdlet. He was able to sign the scripts just fine, but when using the timestamp option the cmdlet was not honoring Internet Explorer Proxy settings and the timestamping process failed.

If you have never seen how to add such a signature with PowerShell, check the following sample:

First we can create a test certificate in i.e. a cmd.exe:

c:\>makecert -n "CN=PowerShell Local Certificate Root" -a sha1 -eku 1.3.6.1.5.5.7.3.3 -r -sv root.pvk root.cer -ss Root -sr localMachine
c:\>makecert -pe -n "CN=Powershell User" -ss MY -a sha1 -eku 1.3.6.1.5.5.7.3.3 -iv root.pvk -ic root.cer

Then we can create a signature with timestamp in PowerShell:

C:\PS>$cert=Get-ChildItem -Path cert:\CurrentUser\my -CodeSigningCert
C:\PS>Set-AuthenticodeSignature -filepath c:\notepad.exe -certificate $cert -IncludeChain All -TimeStampServer "http://timestamp.globalsign.com/scripts/timstamp.dll"

Additionally, remember that we have several other ways to do Authenticode signing as I already commented here:

How to sign EXE files with an Authenticode certificate (VB.NET)

How to sign EXE files with an Authenticode certificate (part 2)

So my customer tried to use signtool.exe instead, with the same results: IE proxy settings are not getting used at all and they get an error like the following:

C:\Program Files\Microsoft SDKs\Windows\v6.1\Bin>signtool timestamp /v /t http://timestamp.globalsign.com/scripts/timstamp.dll c:\notepad.exe

Timestamping: c:\notepad.exe
SignTool Error: The specified timestamp server could not be reached.
SignTool Error: An error occurred while attempting to timestamp: c:\notepad.exe

Number of files successfully timestamped: 0
Number of errors: 1

Note that if we open Internet Explorer and enter http://timestamp.globalsign.com/scripts/timstamp.dll into the address bar, IE presents the credentials dialog box to send to the proxy and we can access the timestamp server without problems.

The cause of this issue is the following:

All signing methods we saw in my previous posts (SignTool, CAPICOM, etc.) end up using SignerSignEx API to sign, and SignerTimeStampEx API to timestamp the signature. The same applies to PowerShell's Set-AuthenticodeSignature cmdlet.

SignerTimeStampEx will connect to the remote timestamp server in a very simple way, without using any proxy info or user credentials. And this is not configurable in any way.

So I'm afraid all signing methods we have at our disposal suffer from this limitation in the API.

In order to timestamp code, it is necessary to have unrestricted access to the Internet.

I hope this helps.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Locks

2009-09-29T11:19:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 2.

LOCKS

· We can see threads waiting for managed locks:

All managed objects have a syncblock (4 bytes) before their Method Table and properties. It can hold locking information for thread-safe operations. All syncblocks being used are stored in a table. A class like System.Threading.Monitor will use them to synchronize access to resources.

With this command we get the Index of the syncblock in the table, the address of the syncblock, the thread holding it & the object which syncblock we are waiting for. Let’s take a look to this sample:

0:006> !SyncBlk

Index SyncBlock MonitorHeld Recursion Owning Thread Info SyncBlock Owner

19 00358ddc 5 2 0032c848 ca8 0 01a2d45c System.Object

-----------------------------

Total 23

CCW 0

RCW 0

ComClassFactory 0

Free 0

Thread 0 is holding another 2 threads (MonitorHeld = 1 for each owner and 2 for each waiter) with a synchronization object.

We can analyze the thread call stacks and see which threads may be held by thread 0:

0:006> ~*kpL

...

4 Id: 1868.ac Suspend: 1 Teb: 7ffdb000 Unfrozen

ChildEBP RetAddr

0461e89c 77550690 ntdll!KiFastSystemCallRet(void)

0461e8a0 76207e09 ntdll!ZwWaitForMultipleObjects(void)+0xc

0461e93c 79ed98fd KERNEL32!WaitForMultipleObjectsEx(unsigned long nCount = 0x461e8f0, void ** lpHandles = 0x00358df0, int bWaitAll = 0, unsigned long dwMilliseconds = 0xffffffff, int bAlertable = 1)+0x11d

0461e9a4 79ed9889 mscorwks!WaitForMultipleObjectsEx_SO_TOLERANT(unsigned long nCount = 1, void ** lpHandles = 0x00358df0, int bWaitAll = 0, unsigned long dwMilliseconds = 0xffffffff, int bAlertable = 1)+0x6f

0461e9c4 79ed9808 mscorwks!Thread::DoAppropriateAptStateWait(int numWaiters = 1, void ** pHandles = 0x00358df0, int bWaitAll = 0, unsigned long timeout = 0xffffffff, WaitMode mode = WaitMode_Alertable (1))+0x3c

0461ea48 79ed96c4 mscorwks!Thread::DoAppropriateWaitWorker(int countHandles = 1, void ** handles = 0x00358df0, int waitAll = 0, unsigned long millis = 0xffffffff, WaitMode mode = WaitMode_Alertable (1))+0x13c

0461ea98 79ed9a62 mscorwks!Thread::DoAppropriateWait(int countHandles = 1, void ** handles = 0x00358df0, int waitAll = 0, unsigned long millis = 0xffffffff, WaitMode mode = WaitMode_Alertable (1), struct PendingSync * syncState = 0x00000000)+0x40

0461eaf4 79e78944 mscorwks!CLREvent::WaitEx(unsigned long dwMilliseconds = 0xffffffff, WaitMode mode = WaitMode_Alertable (1), struct PendingSync * syncState = 0x00000000)+0xf7

0461eb08 79ed7b37 mscorwks!CLREvent::Wait(unsigned long dwMilliseconds = 0xffffffff, int alertable = 1, struct PendingSync * syncState = 0x00000000)+0x17

0461eb94 79ed7a9e mscorwks!AwareLock::EnterEpilog(class Thread * pCurThread = 0x00386aa0, int timeOut = -1)+0x8c

0461ebb0 79ebd7e4 mscorwks!AwareLock::Enter(void)+0x61

0461ec50 0080201c mscorwks!JIT_MonEnterWorker_Portable(class Object * obj = 0x00386aa0)+0xb3

0461ec70 793b0d1f WindowsApplication1!WindowsApplication1.Form1.Run8(<HRESULT 0x80004001>)+0x24

...

5 Id: 1868.1fa8 Suspend: 1 Teb: 7ffd9000 Unfrozen

ChildEBP RetAddr

0446eac4 77550690 ntdll!KiFastSystemCallRet(void)

0446eac8 76207e09 ntdll!ZwWaitForMultipleObjects(void)+0xc

0446eb64 79ed98fd KERNEL32!WaitForMultipleObjectsEx(unsigned long nCount = 0x446eb18, void ** lpHandles = 0x00358df0, int bWaitAll = 0, unsigned long dwMilliseconds = 0xffffffff, int bAlertable = 1)+0x11d

0446ebcc 79ed9889 mscorwks!WaitForMultipleObjectsEx_SO_TOLERANT(unsigned long nCount = 1, void ** lpHandles = 0x00358df0, int bWaitAll = 0, unsigned long dwMilliseconds = 0xffffffff, int bAlertable = 1)+0x6f

0446ebec 79ed9808 mscorwks!Thread::DoAppropriateAptStateWait(int numWaiters = 1, void ** pHandles = 0x00358df0, int bWaitAll = 0, unsigned long timeout = 0xffffffff, WaitMode mode = WaitMode_Alertable (1))+0x3c

0446ec70 79ed96c4 mscorwks!Thread::DoAppropriateWaitWorker(int countHandles = 1, void ** handles = 0x00358df0, int waitAll = 0, unsigned long millis = 0xffffffff, WaitMode mode = WaitMode_Alertable (1))+0x13c

0446ecc0 79ed9a62 mscorwks!Thread::DoAppropriateWait(int countHandles = 1, void ** handles = 0x00358df0, int waitAll = 0, unsigned long millis = 0xffffffff, WaitMode mode = WaitMode_Alertable (1), struct PendingSync * syncState = 0x00000000)+0x40

0446ed1c 79e78944 mscorwks!CLREvent::WaitEx(unsigned long dwMilliseconds = 0xffffffff, WaitMode mode = WaitMode_Alertable (1), struct PendingSync * syncState = 0x00000000)+0xf7

0446ed30 79ed7b37 mscorwks!CLREvent::Wait(unsigned long dwMilliseconds = 0xffffffff, int alertable = 1, struct PendingSync * syncState = 0x00000000)+0x17

0446edbc 79ed7a9e mscorwks!AwareLock::EnterEpilog(class Thread * pCurThread = 0x0038ec10, int timeOut = -1)+0x8c

0446edd8 79f59024 mscorwks!AwareLock::Enter(void)+0x61

0446ee40 79fc6352 mscorwks!AwareLock::Contention(int timeOut = -1)+0x199

0446eee0 0080201c mscorwks!JITutil_MonContention(class AwareLock * lock = 0x00358ddc)+0xa3

0446ef00 793b0d1f WindowsApplication1!WindowsApplication1.Form1.Run8(<HRESULT 0x80004001>)+0x24

...

0:006> ~4e !CLRStack -p

OS Thread Id: 0xac (4)

ESP EIP

0461eb30 77550f34 [GCFrame: 0461eb30]

0461ec00 77550f34 [HelperMethodFrame_1OBJ: 0461ec00] System.Threading.Monitor.Enter(System.Object)

0461ec58 0080201c WindowsApplication1.Form1.Run8()

...

So thread 5 is indeed waiting for our problematic syncblock, and I guess thread 4, too. But I couldn’t see any reference to the syncblock or the related object in the parameters of the call stack of thread 4. Well, the object we used with Monitor.Enter should be near the top of the stack of the thread, as we passed it as a parameter to the managed function at the top of the managed call stack:

0:006> ~4e !dso

OS Thread Id: 0xac (4)

ESP/REG Object Name

0461eb68 01a2d45c System.Object

0461ebc0 01a2d45c System.Object

0461ebd4 01a2d45c System.Object

0461ec30 01a669a4 System.Threading.ThreadStart

0461ec3c 01a2d45c System.Object

0461ec58 01a669fc System.Threading.ThreadHelper

...

And indeed it is, so both threads 4 & 5 are the ones being held by thread 0.

· We can see threads waiting for unmanaged locks:

A class like System.Threading.Mutex is not based on syncblocks, but on kernel mutexes instead. When not working with syncblocks, we can use the same unmanaged commands we’ll use with any unmanaged app:

0:006> !locks

0:006> !critsec

0:006> !SIEExtPub.critlist

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Note: this is the last post on the MANAGED DEBUGGING with WINDBG series. I hope my posts help you the same way they help me in my everyday work.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 2

2009-09-29T11:07:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 1.

BREAKING ON AN EXCEPTION. Part 2

What happens if !PrintException doesn’t return anything, or when we break (i.e. with a breakpoint or manually – Ctrl+Break) we can’t see mscorwks!RaiseTheExceptionInternalOnly in the call stack? What if we already missed the exception when we break?

We could still find the exception near the top of the managed stack of the current thread:

0:000> !DumpStackObjects

...

0:000> !dso

OS Thread Id: 0x15e4 (0)

ESP/REG Object Name

0020eb50 0195fe94 System.IndexOutOfRangeException

0020eb98 0195fe94 System.IndexOutOfRangeException

0020ebdc 01929a04 WindowsApplication1.Form1

0020ebe0 01941958 System.Int32[]

...

0:000> !pe 0x0195fe94

...

If not, we can try to find significant exceptions in the managed heap, as all recent exceptions are stored there:

0:000> !DumpHeap -type Exception

Address MT Size

01991024 790fe044 72

0199106c 790fe0e0 72

019910b4 790fe17c 72

019910fc 790fe284 72

01991144 790fe284 72

0199f848 79103c58 12

0199f860 79103ca4 12

019df490 79117f8c 72

019fc5c8 79117f8c 72

total 9 objects

Statistics:

MT Count TotalSize Class Name

79103ca4 1 12 System.Text.DecoderExceptionFallback

79103c58 1 12 System.Text.EncoderExceptionFallback

790fe17c 1 72 System.ExecutionEngineException

790fe0e0 1 72 System.StackOverflowException

790fe044 1 72 System.OutOfMemoryException

79117f8c 2 144 System.IndexOutOfRangeException

790fe284 2 144 System.Threading.ThreadAbortException

Total 9 objects

0:000> !DumpHeap -mt 79117f8c

Address MT Size

019df490 79117f8c 72

019fc5c8 79117f8c 72

total 2 objects

Statistics:

MT Count TotalSize Class Name

79117f8c 2 144 System.IndexOutOfRangeException

Total 2 objects

0:000> !pe 019df490

...

0:000> !pe 019fc5c8

...

We can also do the following to find all exceptions of a given type in the managed heap:

0:000> .foreach(ex {!dumpheap -type System.IndexOutOfRangeException -short}){!pe ex;.echo *************}

Exception object: 019df490

Exception type: System.IndexOutOfRangeException

Message: Index was outside the bounds of the array.

InnerException: <none>

StackTrace (generated):

SP IP Function

0017EE80 00861AAD WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(Int32[])+0x55

StackTraceString: <none>

HResult: 80131508

*************

Exception object: 019fc5c8

Exception type: System.IndexOutOfRangeException

Message: Index was outside the bounds of the array.

InnerException: <none>

StackTrace (generated):

<none>

StackTraceString: <none>

HResult: 80131508

*************

Note that exception objects are normally created when they are needed. But there are a few exception objects which are created when process starts as they can’t be created when they need to be raised: OutOfMemoryException, ExecutionEngineException, StackOverflowException. So we will always see these exceptions in the heap. That doesn’t mean that they were ever raised.

· We can inspect the method where we got the exception:

We can get i.e. the source code line which failed:

0:000> !pe 019df490

Exception object: 019df490

Exception type: System.IndexOutOfRangeException

Message: Index was outside the bounds of the array.

InnerException: <none>

StackTrace (generated):

SP IP Function

0017EE80 00861AAD WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(Int32[])+0x55

StackTraceString: <none>

HResult: 80131508

0:000> !u 00861AAD

Normal JIT generated code

WindowsApplication1.Form1.PlayWithArray(Int32[])

Begin 00861a58, size c0

00861a58 55 push ebp

...

00861aa3 3b5104 cmp edx,dword ptr [ecx+4]

00861aa6 7205 jb WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(Int32[])+0x55 (00861aad)

00861aa8 e8d67e8c79 call mscorwks!JIT_RngChkFail (7a129983)

>>> 00861aad 03449108 add eax,dword ptr [ecx+edx*4+8]

00861ab1 7105 jno WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(Int32[])+0x60 (00861ab8)

00861ab3 e8837f8c79 call mscorwks!JIT_Overflow (7a129a3b)

...

00861b16 5d pop ebp

00861b17 c3 ret

0:000> u 00861aa8

WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(Int32[])+0x50 [C:\__WORKSHOP\Demos\BuggyNETApp\Form1.vb @ 135]:

00861aa8 e8d67e8c79 call mscorwks!JIT_RngChkFail (7a129983)

...

Note that not all exceptions we saw have the StackTrace field set. I’ve seen that when the debugger just breaks on an exception, the field is empty. Once we go beyond that point and we i.e. enter the “catch” section of the “try{}catch(){}” statement in the code, that field is set.

But if we just broke on an exception, we can still see the call stack with k, !CLRStack, !DumpStack and related commands as we saw in the Call Stack section before, and get the instruction pointer (IP) we need from there.

· We can stop breaking on CLR exceptions:

If we don’t want to break on any other CLR exception we do the following:

0:000> sxn clr

If we were breaking on specific exception types (!StopOnException) before, and we stop breaking on all CLR exceptions, we will still see those exceptions in the debugger:

0:000> g

(1728.1f58): C++ EH exception - code e06d7363 (first chance)

(1728.1f58): CLR exception - code e0434f4d (first chance)

'System.IndexOutOfRangeException hit'

Next post: MANAGED DEBUGGING with WINDBG. Locks.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)

SubAuthentication package gets Zero LogonId

2009-09-01T16:08:00Z

Hi all,

The other day I worked on an issue related to a SubAuthentication package that one of my customers had developed. One of the things that package was doing was getting the LogonId field of the LogonInformation struct being passed to Msv1_0SubAuthenticationFilter method by the Authentication package.

This SubAuth package had been working fine on i.e. Windows XP, but when doing interactive logon on a Windows Server 2003 Domain Controller, LogonId field was always set to Zero. But doing i.e. Network logon on the same DC returned a valid LogonId value. Why?

I debugged the issue and I found out what was going on.

SubAuth packages can only get called by the following Authentication packages: Kerberos or MSV1_0/NTLM.

MSV1_0 (MICROSOFT_AUTHENTICATION_PACKAGE_V1_0) package will provide our SubAuth dll with LogonId info, while Kerberos package won’t. Kerberos ignores LogonId on purpose. This is by design and happens in all versions of Windows.

So every time we get a valid LogonId, it is because logon is taking place through MSV1_0 Auth package. Note that LogonId is just a local ID on the machine and it's not of actual use for SubAuth packages. The purpose of SubAuth packages is not for capturing or tracking logon activities.

SubAuth package supplements part of the authentication and validation criteria used by the main Auth package. It can enforce additional restrictions to the authentication sequence when the DLL is registered under Auth0 registry value for MSV1_0 or Kerberos.

SubAuth package (Msv1_0SubAuthenticationFilter) will get called only after the corresponding Windows logon authentication is successful. The built-in Auth packages get first chance at logon authentication, before SubAuth DLL code gets invoked.

For domain user logon authentication, SubAuth DLLs installed on the DCs of that domain will be called. For local user logon authentication, SubAuth DLLs installed on that member machine will be called.

If you are simply tracking logon events, you would need to enable Auditing Account Logon and look for events logged in the Security Event Log. Audit logon events and Audit account logon events provide tracking of logons at workstations, servers, and DCs (see Audit Account Logon Events for details).

Then an application may get those events programmatically with i.e. WMI and its Win32_NTLogEvent class. I’ve seen people using this idea, for instance here.

There is one thing left to explain. Why is Kerberos always getting used on the DC when doing Interactive logon? We've already seen the Auth packages that can get used, and according to MSDN:

Kerberos
"
The Kerberos authentication package is used when logging on to a network; local logons are handled by MSV1_0.
"

MSV1_0
"
MSV1_0 also supports domain logons. MSV1_0 processes domain logons using pass-through authentication
"

Summing up, MSV1_0 deals with local logons and both MSV1_0 and Kerberos with network logons.

Now, whether Negotiate/Kerberos or NTLM/MSV1_0 is used for network logon depends on the client machine and the component that is doing the authentication. Windows logon components will always attempt to use Kerberos if they can, if there is an SPN (Service Principal Name) for the target name specified. For example, if you are accessing a server say MYSERVER that requires windows authentication and you want to use Kerberos, you have to supply a name that matches an SPN defined on the server account such as MYSERVER.MYDOMAIN.COM. Just specifying MYSERVER may use NTLM if there is no matching SPN. If there is no matching SPN, Kerberos will fall-back to NTLM.

So in my customer's environment we ended up using NTLM/MSV1_0 with network logons and most of local logons, thus getting a valid LogonId thanks to MSV1_0 Auth package.

Now, the exception for local logons are the DCs. Local logon is for local accounts only and not for AD (Active Directory) accounts. Doing an Interactive logon on a DC is not considered a local logon. Because of that we try to use Negotiate/Kerberos first as expected, and we just succeed. And we don't get LogonId because Kerberos Auth package won't provide us with it.

I hope this helps.
Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 1

2009-08-24T12:57:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Managed Heap. Part 5.

BREAKING ON AN EXCEPTION. Part 1

We can only break on exceptions when doing live debugging, but many of the commands explained here can be used when doing dump analysis, too.

· Exceptions we may get in .NET applications:

§ CLR exception (0xe0434f4d).

§ Stack Overflow exception (0x800703e9).

§ Access Violation (0xc0000005).

§ Integer divide-by-zero exception (0xc0000094).

§ ...

§ Fatal Execution Engine Error (<address of failure>). This normally indicates a bug in CLR or NT Heap corruption when doing P/Invoke with not big enough buffers. The address associated to this exception will tell us where in the code the issue happened.

The debugger will break by default when the process raises most of these exception types, but not on CLR exceptions, for instance.

In ASP.NET, if exceptions are raised in managed threads and they are unhandled, Global Error Handle catches them. But if they are raised in another thread like Timer or Finalizer threads, we will crash the process (it will be recycled).

· We can break on any CLR exception:

When our code raises a CLR exception, we see the following line in the debugger:

(1728.1f58): CLR exception - code e0434f4d (first chance)

Unmanaged world doesn’t know about specific CLR exception types, so all of them are raised with the same exception code: 0xe0434f4d.

As we’ve seen, WinDbg won’t break on any CLR exception by default, as they may be expected. We can break on all CLR exceptions with any of these commands:

0:000> sxe clr

0:000> sxe 0xe0434f4d

And when we break we’ll see something like this:

0:000> g

(1728.1f58): C++ EH exception - code e06d7363 (first chance)

(1728.1f58): CLR exception - code e0434f4d (first chance)

First chance exceptions are reported before any exception handling.

This exception may be expected and handled.

eax=002de8e0 ebx=e0434f4d ecx=00000001 edx=00000000 esi=002de968 edi=0047c600

eip=77a1b09e esp=002de8e0 ebp=002de930 iopl=0 nv up ei pl nz ac po nc

cs=001b ss=0023 ds=0023 es=0023 fs=003b gs=0000 efl=00000212

KERNEL32!RaiseException+0x58:

77a1b09e c9 leave

· We can break on a specific CLR exception type:

If we know the exact exception type, we can do the following:

0:004> !StopOnException -create System.IndexOutOfRangeException 1

Breakpoint set

0:004> !soe -create System.IndexOutOfRangeException 1

Breakpoint set

And when we break we’ll see something like this:

0:000> g

(1728.1f58): C++ EH exception - code e06d7363 (first chance)

(1728.1f58): CLR exception - code e0434f4d (first chance)

'System.IndexOutOfRangeException hit'

First chance exceptions are reported before any exception handling.

This exception may be expected and handled.

...

77a1b09e c9 leave

We can also stop only on those exceptions which type derives from a given one:

0:004> !soe -derived -create System.SystemException 1

Breakpoint set

· When we break, we can inspect the CLR exception:

Let’s take a look to the call stack of the thread which raised the exception when the debugger breaks:

0:000> kpL

ChildEBP RetAddr

002de930 79f071ac KERNEL32!RaiseException(unsigned long dwExceptionCode = 0xe0434f4d, unsigned long dwExceptionFlags = 1, unsigned long nNumberOfArguments = 1, unsigned long * lpArguments = 0x002de968)+0x58

002de990 79f9293a mscorwks!RaiseTheExceptionInternalOnly(class Object * throwable = 0x01940e4c, int rethrow = 0, int fForStackOverflow = 0)+0x2a8

002de9c8 7a129a34 mscorwks!UnwindAndContinueRethrowHelperAfterCatch(class Frame * pEntryFrame = 0x002dea20, class Exception * pException = 0x004d2aa8)+0x70

002dea68 00b81aad mscorwks!JIT_RngChkFail(void)+0xb0

002deab0 00b819fe WindowsApplication1!WindowsApplication1.Form1.PlayWithArray(<HRESULT 0x80004001>)+0x55

KERNEL32!RaiseException raises the generic CLR exception (0xe0434f4d) to the unmanaged world, but the specific exception we are getting is the one that mscorwks!RaiseTheExceptionInternalOnly raises. We can inspect that specific exception as any other .NET object:

0:000> !DumpObj 0x0195fe94

...

0:000> !do 0x0195fe94

Name: System.IndexOutOfRangeException

MethodTable: 79117f8c

EEClass: 79117f1c

Size: 72(0x48) bytes

(C:\Windows\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\mscorlib.dll)

Fields:

MT Field Offset Type VT Attr Value Name

790fd8c4 40000b5 4 System.String 0 instance 00000000 _className

7910ebc8 40000b6 8 ...ection.MethodBase 0 instance 00000000 _exceptionMethod

790fd8c4 40000b7 c System.String 0 instance 00000000 _exceptionMethodString

790fd8c4 40000b8 10 System.String 0 instance 01935108 _message

79116114 40000b9 14 ...tions.IDictionary 0 instance 00000000 _data

790fdf04 40000ba 18 System.Exception 0 instance 00000000 _innerException

790fd8c4 40000bb 1c System.String 0 instance 00000000 _helpURL

790fd0f0 40000bc 20 System.Object 0 instance 00000000 _stackTrace

790fd8c4 40000bd 24 System.String 0 instance 00000000 _stackTraceString

...

79102290 40000c1 38 System.Int32 0 instance -2146233080 _HResult

790fd8c4 40000c2 30 System.String 0 instance 00000000 _source

...

0:000> !do 01935108

Name: System.String

MethodTable: 790fd8c4

EEClass: 790fd824

Size: 102(0x66) bytes

(C:\Windows\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\mscorlib.dll)

String: Index was outside the bounds of the array.

Fields:

...

Or we can inspect the exception in a more user friendly way:

0:000> !PrintException 0x0195fe94

...

0:000> !pe 0x0195fe94

Exception object: 0195fe94

Exception type: System.IndexOutOfRangeException

Message: Index was outside the bounds of the array.

InnerException: <none>

StackTrace (generated):

<none>

StackTraceString: <none>

HResult: 80131508

We can save some time if we directly inspect the last thrown exception on the current thread:

0:000> !pe

Exception object: 0195fe94

Exception type: System.IndexOutOfRangeException

Message: Index was outside the bounds of the array.

InnerException: <none>

StackTrace (generated):

<none>

StackTraceString: <none>

HResult: 80131508

Next post: MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 2.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Managed Heap. Part 5

2009-08-24T12:50:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Managed Heap. Part 4.

MANAGED HEAP. Part 5

· We can troubleshoot Loader Heap issues:

An assembly can’t be unloaded while the AppDomain where it was created is still alive. We will get a memory leak if we keep creating new assemblies continuously. The problem is that some .NET operations result on new assemblies being created behind the scenes:

§ Create a new XmlSerializer with any constructor but XmlSerializer(type) or XmlSerializer(type, defaultnamespace).

Memory usage is high when you create several XmlSerializer objects in ASP.NET

§ Make XSLT transformations where XSLT Transform contains a script.

PRB: Cannot unload assemblies that you create and load by using script in XSLT

§ Change ASPX, ASCX or ASMX pages which cause re-compilations.

This could be avoided by caching XMLSerializers and XSLT Transforms, and using maxNumRecompiles value which is set to 15 by default.

Imagine that we’ve detected a memory leak in our ASP.NET app, and PerfMon shows that during the time of the memory leak “.NET CLR Loading:Bytes in loader heap” is constantly increasing. The same thing is true for “.NET CLR Loading:Current Assemblies”. We check the number of assemblies in the heap:

0:000> !DumpHeap -stat -type Assembly

------------------------------

Heap 0

total 1141 objects

------------------------------

Heap 1

total 1220 objects

------------------------------

total 2361 objects

Statistics:

MT Count TotalSize Class Name

7912ad0c 1 24 System.Collections.Generic.List`1[[System.Reflection.Assembly, mscorlib]]

...

790fc79c 2297 55128 System.Reflection.Assembly

Total 2361 objects

And we try to get some info about those assemblies:

0:000> !DumpDomain

...

--------------------------------------

Domain 2: 001cb1d8

LowFrequencyHeap: 001cb1fc

HighFrequencyHeap: 001cb258

StubHeap: 001cb2b4

Stage: OPEN

SecurityDescriptor: 001c9ab0

Name: /LM/w3svc/1/ROOT/DebuggingWorkshop-1-128030642795871345

Assembly: 001b6660 [C:\WINDOWS\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\mscorlib.dll]

ClassLoader: 001773b8

SecurityDescriptor: 642a2e7e

Module Name

790c2000 C:\WINDOWS\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\mscorlib.dll

02272380 C:\WINDOWS\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\sortkey.nlp

02272010 C:\WINDOWS\assembly\GAC_32\mscorlib\2.0.0.0__b77a5c561934e089\sorttbls.nlp

...

Assembly: 0020b220 [C:\WINDOWS\assembly\GAC_MSIL\System.Drawing\2.0.0.0__b03f5f7f11d50a3a\System.Drawing.dll]

ClassLoader: 0020c808

SecurityDescriptor: 642a2e7e

Module Name

7ae72000 C:\WINDOWS\assembly\GAC_MSIL\System.Drawing\2.0.0.0__b03f5f7f11d50a3a\System.Drawing.dll

Assembly: 0020e778 [3k2vmufw, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null]

ClassLoader: 0020e800

SecurityDescriptor: 642a2e7e

Module Name

02837b58 3k2vmufw, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null

Assembly: 0021c050 [zxsplww6, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null]

ClassLoader: 0021c0d8

SecurityDescriptor: 642a2e7e

Module Name

02837f34 zxsplww6, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null

...

Assembly: 14bdbf88 [oy_7tsbs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null]

ClassLoader: 14bdc010

SecurityDescriptor: 642a2e7e

Module Name

19a273d8 oy_7tsbs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null

0:000> !DumpModule 19a273d8

Name: oy_7tsbs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null

Attributes: PEFile

Assembly: 14bdbf88

LoaderHeap: 00000000

TypeDefToMethodTableMap: 19bf6158

TypeRefToMethodTableMap: 19bf6170

MethodDefToDescMap: 19bf61dc

FieldDefToDescMap: 19bf6250

MemberRefToDescMap: 19bf62b8

FileReferencesMap: 19bf63fc

AssemblyReferencesMap: 19bf6400

MetaData start address: 19c312d8 (6176 bytes)

0:000> dc 19c312d8 19c312d8+6176

19c312d8 424a5342 00010001 00000000 0000000c BSJB............

...

19c319e8 00010000 0000039b 00000000 6f4d3c00 .............<Mo

19c319f8 656c7564 796f003e 7374375f 642e7362 dule>.oy_7tsbs.d

19c31a08 58006c6c 65536c6d 6c616972 74617a69 ll.XmlSerializat

19c31a18 576e6f69 65746972 72755072 73616863 ionWriterPurchas

19c31a28 64724f65 4d007265 6f726369 74666f73 eOrder.Microsoft

19c31a38 6c6d582e 7265532e 696c6169 6974617a .Xml.Serializati

19c31a48 472e6e6f 72656e65 64657461 65737341 on.GeneratedAsse

19c31a58 796c626d 6c6d5800 69726553 7a696c61 mbly.XmlSerializ

19c31a68 6f697461 6165526e 50726564 68637275 ationReaderPurch

19c31a78 4f657361 72656472 6c6d5800 69726553 aseOrder.XmlSeri

...

We check our code and we can see this:

protected void Page_Load(object sender, EventArgs e)

{

XmlSerializer mySerializer = new XmlSerializer(typeof(PurchaseOrder), new XmlRootAttribute("order"));

...

}

We can see how much memory is taken up by the Loader heap:

0:000> !EEHeap -loader

Loader Heap:

...

--------------------------------------

Module Thunk heaps:

Module 648e8000: Size: 0x0(0)bytes.

Module 68a06000: Size: 0x0(0)bytes.

...

Module 19a273d8: Size: 0x0(0)bytes.

Total size: 0x0(0)bytes

--------------------------------------

Total LoaderHeap size: 0xe33000(14888960)bytes

=======================================

0:000> !DumpModule 19a273d8

Name: oy_7tsbs, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null

...

Next post: MANAGED DEBUGGING with WINDBG. Breaking on an Exception. Part 1.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Managed Heap. Part 4

2009-08-17T09:32:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Managed Heap. Part 3.

MANAGED HEAP. Part 4

· We can troubleshoot Heap issues:

These are some of the important performance counters we can check with Performance Monitor when having memory issues with .NET applications:

§ Process:Private Bytes: All the memory that the app is using (committed, could be paged). When this value reaches Virtual Bytes, Virtual Bytes doubles its size. If memory is shared, it won’t count as Private Bytes, but as Working Set.

§ Process:Virtual Bytes: Virtual Memory of a process. Memory can be reserved or used (committed, reflected as Private Bytes). This counter is useful when we get Out of Memory exceptions.

§ Process:Working Set: How much memory is paged in RAM.

§ .NET CLR Memory:#bytes in all heaps: .NET won’t use the NT Heap, but saves some virtual memory for its own heap. If this value doesn’t grow but Virtual Bytes do, the memory leak is in the unmanaged world.

§ .NET CLR Memory:#gen 0/1/2 collections.

§ .NET CLR Memory:Gen 0/1/2 Heap Size.

§ .NET CLR Memory:Large Object Heap Size.

§ .NET CLR Memory:# induced GC: Number of calls to GC.Collect. Calling this function in our code is not recommended, as GC auto-adjust for better performance.

§ .NET CLR Memory:# of pinned objects: Number of objects which reference has been blocked (“pinned”) in memory. GC won’t be able to move pinned objects around memory to optimize available space in heap.

§ .NET CLR Memory:finalization survivors: Number of objects which survived a GC because of having a Finalizer method.

§ .NET CLR Memory:Promoted Finalization Memory from Gen 0/1.

§ .NET CLR Memory:Promoted Memory from Gen 0/1.

§ .NET CLR Memory:% Time in GC.

§ .NET CLR Loading:Bytes in Loader Heap: A continuous increase in the size of the Loader Heap is typically an effect of extensive usage of dynamic modules (.NET CLR Loading:Current Assemblies) or having “debug=true” in ASP.NET apps.

§ .NET CLR Loading:Current Assemblies.

§ .NET CLR Loading:Current appdomains.

Additionally, we can generate a log file which can be used to troubleshoot managed memory usage issues. We can take a look to this log file with CLR Profiler:

0:004> !TraverseHeap c:\memory.log

Writing CLRProfiler format to file c:\memory.log

Gathering types...

tracing roots...

Scan Thread 0 OSTHread 1440

Scan Thread 2 OSTHread 194

Walking heap...

....................................................................................................................................................................................

file c:\memory.log saved

We can also check the heap for signs of corruption:

0:004> !VerifyHeap

-verify will only produce output if there are errors in the heap

· We can troubleshoot GCHandle leaks:

Sometimes the source of a memory leak is a GCHandle leak. If we discard a GCHandle which points to i.e. a big array without freeing it, the array will be kept alive.

We can see statistics about GCHandles in the process:

0:004> !GCHandles

GC Handle Statistics:

Strong Handles: 33

Pinned Handles: 8

Async Pinned Handles: 0

Ref Count Handles: 0

Weak Long Handles: 56

Weak Short Handles: 384

Other Handles: 0

Statistics:

MT Count TotalSize Class Name

790fd0f0 1 12 System.Object

...

7b485894 357 22848 System.Windows.Forms.Internal.DeviceContext

7912d8f8 8 28032 System.Object[]

Total 481 objects

We can also try to track down GCHandle leaks. We can search all of memory for any references to the Strong and Pinned GCHandles in the process:

0:004> !GCHandleLeaks

-------------------------------------------------------------------------------

GCHandleLeaks will report any GCHandles that couldn't be found in memory.

Strong and Pinned GCHandles are reported at this time. You can safely abort the

memory scan with Control-C or Control-Break.

-------------------------------------------------------------------------------

Found 41 handles:

001c1118 001c1120 001c1124 001c1148

...

001c13ec 001c13f0 001c13f4 001c13f8

001c13fc

Searching memory

Reference found in stress log will be ignored

Found 001c1188 at location 0030f23c

Found 001c11fc at location 00424fb8

...

Found 001c11d4 at location 7a3b9c88

------------------------------------------------------------------------------

All handles found.

A leak may still exist because in a general scan of process memory SOS can't

differentiate between garbage and valid structures, so you may have false

positives. If you still suspect a leak, use this function over time to

identify a possible trend.

------------------------------------------------------------------------------

Note that if a handle is found, we’ll see the address of the reference. This might be a stack address or a field within an object, for example.

We can see the object a handle references by dereferencing it:

0:004> !do poi(001c1118)

Name: WindowsApplication1.Form1

...

If a handle is not found in memory, we get a notification:

0:004> !GCHandleLeaks

-------------------------------------------------------------------------------

GCHandleLeaks will report any GCHandles that couldn't be found in memory.

Strong and Pinned GCHandles are reported at this time. You can safely abort the

memory scan with Control-C or Control-Break.

-------------------------------------------------------------------------------

Found 249 handles:

01dc1200 01dc1208 01dc1210 01dc1218

...

Searching memory

Found 01dc1ff8 at location 0012ce48

...

------------------------------------------------------------------------------

Some handles were not found. If the number of not-found handles grows over the

lifetime of your application, you may have a GCHandle leak. This will cause

the GC Heap to grow larger as objects are being kept alive, referenced only

by the orphaned handle. If the number doesn't grow over time, note that there

may be some noise in this output, as an unmanaged application may be storing

the handle in a non-standard way, perhaps with some bits flipped. The memory

scan wouldn't be able to find those.

------------------------------------------------------------------------------

Didn't find 232 handles:

01dc1200 01dc1208 01dc1210 01dc1218

...

0:004> !do poi(01dc1218)

...

If a serious leak is going on, we’ll get an ever-growing set of handle addresses that couldn't be found.

Next post: MANAGED DEBUGGING with WINDBG. Managed Heap. Part 5.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Managed Heap. Part 3

2009-08-17T09:26:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Managed Heap. Part 2.

MANAGED HEAP. Part 3

· We can take a look to which objects reference which objects in the heap:

We may see many objects in the heap that we think should have been cleaned up already. If they haven’t, it’s because they are still being referenced and GC can’t clean them.

We can look for references (“roots”) to an object. Let’s see this with a sample taken from an ASP.NET app:

0:000> !DumpHeap -stat

...

Statistics:

MT Count TotalSize Class Name

...

0ee10034 19218 7149096 ASP.lab3_1_aspx

...

Total 1748273 objects

0:000> !DumpHeap -mt 0ee10034

------------------------------

Heap 0

Address MT Size

...

0556b268 0ee10034 372

0556ccac 0ee10034 372

…

total 6524 objects

------------------------------

Heap 1

...

0:000> !GCRoot 0556b268

Note: Roots found on stacks may be false positives. Run "!help gcroot" for

more info.

Scan Thread 15 OSTHread 2c4

...

Scan Thread 13 OSTHread c4c

esi:Root:14f002dc(System.Web.HttpResponse)->

14f00194(System.Web.HttpContext)->

14f00a08(ASP.lab3_1_aspx)->

02b33d80(System.Web.Caching.Cache)->

02b33e60(System.Web.Caching.CacheMultiple)->

02b33e78(System.Object[])->

02b33e90(System.Web.Caching.CacheSingle)->

02b33f38(System.Web.Caching.CacheExpires)->

02b33f58(System.Object[])->

02b34878(System.Web.Caching.ExpiresBucket)->

10872ea0(System.Web.Caching.ExpiresPage[])->

094b5734(System.Web.Caching.ExpiresEntry[])->

0556bb94(System.Web.Caching.CacheEntry)->

0556bb74(System.Web.Caching.CacheItemRemovedCallback)->

0556b268(ASP.lab3_1_aspx)

Scan Thread 22 OSTHread 830

Scan Thread 23 OSTHread f7c

If we take a look to the source code of Lab3_1.aspx page, we can see why the page is being kept in memory by the Cache object:

public partial class Lab3_1 : System.Web.UI.Page

{

public void RemovedCallback(String k, Object v, System.Web.Caching.CacheItemRemovedReason r)

{

//do stuff when the item is removed

}

protected void Page_Load(object sender, EventArgs e)

{

UserInfo u = new UserInfo("James", "Dean", "MainStreet 21, NY");

Cache.Add(Guid.NewGuid().ToString(), u, null, System.Web.Caching.Cache.NoAbsoluteExpiration, new TimeSpan(0, 5, 0), System.Web.Caching.CacheItemPriority.NotRemovable, new System.Web.Caching.CacheItemRemovedCallback(this.RemovedCallback));

}

References can exist in several places:

1. In a registry of a thread:

0:022> !gcroot 0x14ef2a34

…

esi:Root: 14f002dc(System.Web.HttpResponse)->

…->

14ef2a34(ASP.lab3_1_aspx)

2. In the stack of a thread:

ESP:ef4f610:Root: 14f00194(System.Web.HttpContext)->

… ->

14ef2a34(ASP.lab3_1_aspx)

3. Within a GCHandle.

Different kind of handles can reference our objects:

- Strong Handles: they keep the object they point to alive until the handle is explicitly freed. Typically a static variable.

DOMAIN(001CAFF8):HANDLE(Strong):284118c:Root: 02b35d3c(System.Threading._TimerCallback)->

… ->

14ef2a34(ASP.lab3_1_aspx)

- Pinned Handles: they are used to prevent the GC from moving an object during collection. Pinning objects can cause heap fragmentation.

DOMAIN(001CCE68):HANDLE(Pinned) …

- Weak Short Handles: they are non null as long as the object they reference is alive because of another Strong reference. They are null otherwise.

DOMAIN(001CCE68):HANDLE(WeakSh) …

- Weak Long Handles: they are non null as long as the object they reference hasn’t been reclaimed by the GC. It can be resurrected.

DOMAIN(001CCE68):HANDLE(WeakLn) …

The distinction between Weak Short and Weak Long comes from the finalization semantics: a Weak Short reference is null if the object can be freed, but a Weak Long reference still refers to the freachable object.

4. In an object ready for finalization.

The object is no longer in use but nobody called its Dispose method.

Finalizer queue:Root: …

5. As a member of an object found in 1, 2 or 3 above.

If our object is referenced by another one, it can’t be garbage collected. We should get rid of that reference or dispose the object that references ours.

…

14ef3914(System.Web.Caching.CacheEntry)->

14ef38f4(System.Web.Caching.CacheItemRemovedCallback)->

14ef2a34(ASP.lab3_1_aspx)

If there is a live reference to an object, that object is said to be strongly rooted. .NET also introduces the notion of weakly rooted references. A weak reference provides a way for programmers to indicate to the GC that they want to be able to access an object, but they don't want to prevent the object from being collected. Such an object is available until it is collected by the GC. For example, you could allocate a large object, and rather than fully deleting and collecting the object, you could hold onto it for possible reuse, as long as there is no memory pressure to clean up the managed heap. Thus, weak references behave somewhat like a cache.

· We can calculate the size of an object:

We can see the size of an object including its child objects. For instance, to determine how much an ASP.NET app is caching, we can do the following:

0:028> !Name2EE * System.Web.Caching.Cache

...

--------------------------------------

Module: 6638c000 (System.Web.dll)

Token: 0x020000f7

MethodTable: 6639d878

EEClass: 6639d808

Name: System.Web.Caching.Cache

--------------------------------------

...

0:028> !DumpHeap -mt 6639d878

------------------------------

Heap 0

Address MT Size

046b9198 6639d878 12

total 1 objects

------------------------------

Heap 1

Address MT Size

total 0 objects

------------------------------

total 1 objects

Statistics:

MT Count TotalSize Class Name

6639d878 1 12 System.Web.Caching.Cache

Total 1 objects

0:028> !ObjSize 046b9198

sizeof(046b9198) = 319128 ( 0x4de98) bytes (System.Web.Caching.Cache)

Next post: MANAGED DEBUGGING with WINDBG. Managed Heap. Part 4.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)

MANAGED DEBUGGING with WINDBG. Managed Heap. Part 2

2009-08-13T10:47:00Z

Hi all,

This post is a continuation of MANAGED DEBUGGING with WINDBG. Managed Heap. Part 1.

MANAGED HEAP. Part 2

· We can take a look to the objects in the heap:

We can see all objects in the heap:

0:004> !DumpHeap

Address MT Size

7b463c40 790fd0f0 12

7b463c4c 790fd8c4 36

7b463c70 790fd8c4 20

...

01905378 79119a1c 32

...

02907de0 003d6d48 16 Free

total 12400 objects

Statistics:

MT Count TotalSize Class Name

7b492254 1 12 System.Windows.Forms.VisualStyles.VisualStyleRenderer+ThemeHandle

...

79119a1c 3 96 System.Security.Policy.PermissionRequestEvidence

...

003d6d48 11 164 Free

...

790fd0f0 165 1980 System.Object

...

790fd8c4 7308 435252 System.String

Total 12400 objects

0:004> !do 01905378

Name: System.Security.Policy.PermissionRequestEvidence

...

0:004> !do 02907de0

Free Object

Size 16(0x10) bytes

Note the existence of Free objects in the heap. The Free entry represents objects which are not referenced and have been garbage collected, but whose memory hasn't been compacted and released to the operating system yet. These regions of space can be re-used by the Garbage Collector later. If 30% or more of the heap contains Free objects, the process may suffer from heap fragmentation.

I’m sure we noticed that there are too many objects in the heap, so it’s usually better to see only the statistics:

0:004> !DumpHeap -stat

total 12400 objects

Statistics:

MT Count TotalSize Class Name

7b492254 1 12 System.Windows.Forms.VisualStyles.VisualStyleRenderer+ThemeHandle

...

79119a1c 3 96 System.Security.Policy.PermissionRequestEvidence

...

003d6d48 11 164 Free

...

790fd0f0 165 1980 System.Object

...

790fd8c4 7308 435252 System.String

Total 12400 objects

We can also focus on the objects of a given type only:

0:004> !Name2EE * System.Security.Policy.PermissionRequestEvidence

Module: 790c2000 (mscorlib.dll)

Token: 0x0200048a

MethodTable: 79119a1c

...

0:004> !DumpHeap -mt 79119a1c

Address MT Size

01901ebc 79119a1c 32

01905378 79119a1c 32

019290c0 79119a1c 32

total 3 objects

Statistics:

MT Count TotalSize Class Name

79119a1c 3 96 System.Security.Policy.PermissionRequestEvidence

Total 3 objects

Or focus on objects of several types with part of the name in common:

0:004> !DumpHeap -type System.Security.Policy

Address MT Size

01901b78 7910ed68 20

...

01901ebc 79119a1c 32

...

0192971c 79119bf4 16

total 38 objects

Statistics:

MT Count TotalSize Class Name

791197b0 1 12 System.Security.PolicyManager

7911c924 1 24 System.Security.Policy.Hash

...

79119a1c 3 96 System.Security.Policy.PermissionRequestEvidence

...

790f97c4 3 156 System.Security.Policy.PolicyLevel

Total 38 objects

We can see the objects in the Large Object Heap, and we can do it in several ways:

0:004> !EEHeap -GC

Number of GC Heaps: 1

generation 0 starts at 0x01901018

generation 1 starts at 0x0190100c

generation 2 starts at 0x01901000

ephemeral segment allocation context: none

segment begin allocated size

00423878 7b463c40 7b47a744 0x00016b04(92932)

0041d178 7a733370 7a754b98 0x00021828(137256)

003d6600 790d8620 790f7d8c 0x0001f76c(128876)

01900000 01901000 01949ff4 0x00048ff4(298996)

Large object heap starts at 0x02901000

segment begin allocated size

02900000 02901000 02907df0 0x00006df0(28144)

Total Size 0xa787c(686204)

------------------------------

GC Heap Size 0xa787c(686204)

0:004> !DumpHeap 02901000 02907df0

Address MT Size

02901000 003d6d48 16 Free

02901010 7912d8f8 4096

...

total 15 objects

Statistics:

MT Count TotalSize Class Name

003d6d48 8 128 Free

7912d8f8 7 28016 System.Object[]

Total 15 objects

0:004> !SOS.DumpHeap -min 85000

...

Note that there are objects in LOH which size is smaller than 85,000 bytes (?).

We can also see the objects which are ready for finalization, and in general, which objects in our app have a Finalize method:

0:004> !FinalizeQueue

SyncBlocks to be cleaned up: 0

MTA Interfaces to be released: 0

STA Interfaces to be released: 0

----------------------------------

generation 0 has 703 finalizable objects (00457998->00458494)

generation 1 has 0 finalizable objects (00457998->00457998)

generation 2 has 0 finalizable objects (00457998->00457998)

Ready for finalization 0 objects (00458494->00458494)

Statistics:

MT Count TotalSize Class Name

7b492254 1 12 System.Windows.Forms.VisualStyles.VisualStyleRenderer+ThemeHandle

...

7ae77fa8 2 80 System.Drawing.Icon

...

7b485894 59 3776 System.Windows.Forms.Internal.DeviceContext

Total 703 objects

0:004> dd 00457998 00458494-4

00457998 01901f10 01905238 01906414 01906428

...

00457b28 019323f8 019324b0 0193296c 01936378

...

00458488 01949468 019494f0 0194959c

0:004> !DumpHeap -mt 7ae77fa8

Address MT Size

0193296c 7ae77fa8 40

01936378 7ae77fa8 40

total 2 objects

Statistics:

MT Count TotalSize Class Name

7ae77fa8 2 80 System.Drawing.Icon

Total 2 objects

0:004> !do 0193296c

Name: System.Drawing.Icon

...

0:004> !DumpMT -md 7ae77fa8

...

Name: System.Drawing.Icon

...

--------------------------------------

MethodDesc Table

Entry MethodDesc JIT Name

...

7ae2070c 7aea3c88 PreJIT System.Drawing.Icon.Finalize()

...

7ae201dc 7aea3c58 PreJIT System.Drawing.Icon.Dispose()

...

Look to the way Reflector.exe shows the Dispose and Finalize (destructor) methods of the “Icon” class we saw above, and how Dispose tells the GC to forget about the Finalize method:

public void Dispose()

{

this.Dispose(true);

GC.SuppressFinalize(this);

}

~Icon()

{

this.Dispose(false);

}

We can get extra information on the following objects registered for finalization: SyncBlocks and RuntimeCallableWrappers (RCWs). Both of these data structures are cached and cleaned up by the Finalizer thread.

0:000> !FinalizeQueue -detail

To be cleaned Com Data

0x4edc1b8 ComPlusWrapper

0x4edc590 ComPlusWrapper

...

0x4e227b8 ComPlusWrapper

SyncBlock to be cleaned up: 6371

MTA interfaces to be released: 0

STA interfaces to be released: 0

...

Every time a COM interface pointer enters the CLR, it is wrapped in a RCW. The RCW has a reference count that is incremented every time a COM interface pointer is mapped to it. The System.Runtime.InteropServices.Marshal.ReleaseComObject method decrements the reference count. If the same COM interface is passed more than once from unmanaged to managed code, the reference count on the wrapper is incremented every time and calling ReleaseComObject returns the number of remaining references. When the reference count reaches zero, the runtime releases all its references on the unmanaged COM object. We use this method to free the COM object and the resources it holds.

Next post: MANAGED DEBUGGING with WINDBG. Managed Heap. Part 3.

Index: MANAGED DEBUGGING with WINDBG. Introduction and Index.

Regards,

Alex (Alejandro Campos Magencio)