I recently had a request to provide a SharePoint Administrator the ability to export out a crawl log to a CSV file using PowerShell. Luckily, I found Vijay’s (Thanks Vijay!) post which saved me a ton of time:
http://blogs.msdn.com/b/spses/archive/2011/06/22/exporting-sharepoint-2010-search-crawl-logs.aspx
I wanted to add some more functionality to this process of exporting out to a CSV. I wrote the following PowerShell script which gives the Admin more options like choosing which Search Service Application and filtering by URL or by Content Source.
Updated on 2/18/2012 – Thanks to my colleague Heiko at Microsoft. He pointed out the lack of functionality when multiple crawl stores exists within the Search Service Application. I updated the script to now work with Search Service Applications containing multiple crawl stores.
This script does the following:
1. Let you decide which Search Service Application to use 2. Let you decide if you want to filter against URL or by Content Source 3. If you choose URL, you have an option of only exporting Errors or All Events (Skip to Step 5) 4. If you choose Content Source, you decide which content source to filter against 4.a. After Choosing Content Source, you have an option of only exporting Errors or All Events
5. Finally, you can choose the path and file name to use and the file will export the result set as a CSV which can be opened in Excel
Instructions for running the script: 1. Copy the below script and save it in notepad 2. Save it with a anyfilename.ps1 extension 3. To run, copy the file to a SharePoint Server 4. Select Start\Microsoft SharePoint 2010 Products\SharePoint 2010 Management Shell 5. Browse to directory holding the copied script file 6. Run the script: .\anyfilename.ps1 (assuming anyfilename is the name of the file)
<# ============================================================== // // Microsoft provides programming examples for illustration only, // without warranty either expressed or implied, including, but not // limited to, the implied warranties of merchantability and/or // fitness for a particular purpose. // // This sample assumes that you are familiar with the programming // language being demonstrated and the tools used to create and debug // procedures. Microsoft support professionals can help explain the // functionality of a particular procedure, but they will not modify // these examples to provide added functionality or construct // procedures to meet your specific needs. If you have limited // programming experience, you may want to contact a Microsoft // Certified Partner or the Microsoft fee-based consulting line at // (800) 936-5200. // // For more information about Microsoft Certified Partners, please // visit the following Microsoft Web site: // https://partner.microsoft.com/global/30000104 // // Author: Russ Maxwell (russmax@microsoft.com) // // ---------------------------------------------------------- #>
[Void][System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint")
Start-SPAssignment -Global
############################# #Function to Export the Data# ############################# function exportThis { $output = Read-Host "Enter a location for the output file (For Example: c:\logs\)" $filename = Read-Host "Enter a filename" $stores = $ssa.CrawlStores $storectr = $stores.count $name = $output + "\" + $filename + ".csv" if($storectr -eq '1') { $logViewer = New-Object Microsoft.Office.Server.Search.Administration.Logviewer $ssa $i = 0 $urlOutput = $logViewer.GetCurrentCrawlLogData($crawlLogFilters, ([ref] $i)) Write-Host "# of Crawl Entries Produced" $urlOutput.Rows.Count $urlOutput | Export-Csv $name -NoTypeInformation Write-Host "Your results were exported to: " $name } elseif($storectr -gt '1') { $f = 1 foreach($store in $stores) { Write-Host "In the " $f " iteration of store object" $logViewer = New-Object Microsoft.Office.Server.Search.Administration.Logviewer $store $i = 0 $urlOutput = $logViewer.GetCurrentCrawlLogData($crawlLogFilters, ([ref] $i)) $ctr = $urlOutput.Rows.Count $officialCTR += $ctr if($f -eq '1') { $finalDT = New-Object System.Data.DataTable $finalDT = $urlOutput.Copy() } else { $finalDT.Merge($urlOutput) } $f++ } } $finalDT | Export-Csv $name -NoTypeInformation Write-Host "# of Crawl Entries Produced" $officialCTR }
##################################### #Choose a Search Service Application# ##################################### $ssa = Get-SPEnterpriseSearchServiceApplication $ssaName = $ssa | ForEach-Object {$_.Name} Write-Host "Choose a Search Service Application to review crawl logs" Write-Host $num = 1
Foreach($sa in $ssa) { Write-Host $num $sa.Name $num++ }
Write-Host $result = Read-Host "Enter the number next to the desired Search Service Application and press enter"
$num = 1 Foreach($i in $ssa) { if($num -eq $result) { $ssa = $i } $num++ } Write-Host
############################################### #Create a Logviewer and Crawl Log FilterObject# ############################################### $crawlLogFilters = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilters
###################### #Let the Admin choose# ###################### Write-Host "How would you like to filter the crawl log?" Write-Host "1 Filter Based on a URL" Write-Host "2 Filter Based on Content Source" Write-Host "3 Export without a Filter" Write-Host $choice = Read-Host "Enter 1, 2, or 3 and press enter" Write-Host Write-Host Write-Host "1 Export only errors" Write-Host "2 Export All (Success, Warning, and Errors" $type = Read-Host "Enter 1 or 2 and press enter" Write-Host
if($choice -eq '1') { $url = Read-Host "Enter the URL to filter on" ################################### #Create Property and add to filter# ################################### $totalentryProp = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty $totalentryProp = [Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty]::TotalEntries $urlProp = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty $urlProp = [Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty]::Url $stringOp = New-Object Microsoft.Office.Server.Search.Administration.StringFilterOperator $stringOp = [Microsoft.Office.Server.Search.Administration.StringFilterOperator]::Contains $crawlLogFilters.AddFilter($urlProp, $stringOp,$url) $crawlLogFilters.AddFilter($totalentryProp, "1,000,000") if($type -eq '1') { $typeEnum = New-Object Microsoft.Office.Server.Search.Administration.MessageType $typeEnum = [Microsoft.Office.Server.Search.Administration.MessageType]::Error $crawlLogFilters.AddFilter($typeEnum) } #Calling exportThisfunction exportThis }
elseif($choice -eq '2') { ######################### #Choose a content source# ######################### $content = New-Object Microsoft.Office.Server.Search.Administration.Content($ssa) $contentsources = $content.ContentSources Write-Host "Choose a Content Source to filter on" Write-Host $num = 1
Foreach($c in $contentsources) { Write-Host $num": " $c.Name $num++ }
$result = Read-Host "Enter the associated # press enter"
$num = 1 Foreach($c in $contentsources) { if($num -eq $result) { $contentSource = $c } $num++ } Write-Host "You chose" $contentSource.Name $id = $contentSource.Id ################################### #Create Property and add to filter# ################################### $totalentryProp = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty $totalentryProp = [Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty]::TotalEntries $csProp = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty $csProp = [Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty]::ContentSourceId $crawlLogFilters.AddFilter($csProp, $id) $crawlLogFilters.AddFilter($totalentryProp, "1,000,000") if($type -eq '1') { $typeEnum = New-Object Microsoft.Office.Server.Search.Administration.MessageType $typeEnum = [Microsoft.Office.Server.Search.Administration.MessageType]::Error $crawlLogFilters.AddFilter($typeEnum) } #Calling exportThisfunction# exportThis }
elseif($choice -eq '3') { $catProp = New-Object Microsoft.Office.Server.Search.Administration.CatalogType $catProp = [Microsoft.Office.Server.Search.Administration.CatalogType]::PortalContent $crawlLogFilters.AddFilter($catProp) $catProp2 = New-Object Microsoft.Office.Server.Search.Administration.CatalogType $catProp2 = [Microsoft.Office.Server.Search.Administration.CatalogType]::ProfileContent $crawlLogFilters.AddFilter($catProp2) $totalentryProp = New-Object Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty $totalentryProp = [Microsoft.Office.Server.Search.Administration.CrawlLogFilterProperty]::TotalEntries $crawlLogFilters.AddFilter($totalentryProp, "1,000,000") if($type -eq '1') { $typeEnum = New-Object Microsoft.Office.Server.Search.Administration.MessageType $typeEnum = [Microsoft.Office.Server.Search.Administration.MessageType]::Error $crawlLogFilters.AddFilter($typeEnum) } #Calling exportThisfunction exportThis } Stop-SPAssignment –Global
Thanks!
Russ Maxwell, MSFT
I apologize it’s been so long since I’ve published any new blog content. I recently moved into a new position in Microsoft as a Premier Field Engineer. I’m really enjoying it so far and will start pumping out new blog content soon. For now, I recently wrote a PowerShell script which fetches the item count for items that reside in all Document Libraries for a given Site Collection. That includes all Document Libraries that exists in both the root site and all sub sites.
The script will produce the following output:
1. Total # of Document Libraries with 0 items 2. Total # of Document Libraries with 1 or more items 2. Total # of items in all Document Libraries 3. Total # of items in DocLib\Subfolders 4. Start Time and End Time of Script
Script is below:
<# ============================================================== // Microsoft provides programming examples for illustration only, // without warranty either expressed or implied, including, but not // limited to, the implied warranties of merchantability and/or // fitness for a particular purpose. // // This sample assumes that you are familiar with the programming // language being demonstrated and the tools used to create and debug // procedures. Microsoft support professionals can help explain the // functionality of a particular procedure, but they will not modify // these examples to provide added functionality or construct // procedures to meet your specific needs. // Author: Russ Maxwell (russmax@microsoft.com) // ---------------------------------------------------------- #>
[Void][System.Reflection.Assembly]::LoadWithPartialName("Microsoft.SharePoint") Start-SPAssignment -Global
$starttime = Get-Date #Creating new site object $siteurl = Read-Host "Enter the name of your site and press enter" $site = New-Object Microsoft.SharePoint.SPSite($siteurl)
#Assigning all webs (sites) to $webs $webs = $site.Allwebs
#################################################### #System Libraries defined so they won't be touched## ####################################################
$systemlibs =@("Converted Forms", "Customized Reports", "Documents", "Form Templates", "Images", "List Template Gallery", "Master Page Gallery", "Pages", "Reporting Templates", "Site Assets", "Site Collection Documents", "Site Collection Images", "Site Pages", "Solution Gallery", "Style Library", "Theme Gallery", "Web Part Gallery", "wfpub")
Write-Host "Total number of webs that will be traversed: " $webs.count
$DocLibsCount = 0 $DocLibwItems = 0 $totalitems = 0 $subfolderitems = 0
foreach($web in $webs) { $listcoll = $web.lists foreach($list in $listcoll) { if($list -eq $null) { Write-Host } else { $base = $list.GetType() if($base.name -eq "SPDocumentLibrary") { if ($systemlibs -contains $list) { continue} else { $DocLibsCount += 1 $items = $list.items if($items -ne "0") { $DocLibwItems += 1 Write-Host "Processing ItemCount for DobLib " $DocLibsCount -ForegroundColor Red $totalitems += $items.count $name = $list.Title $folders = $web.GetFolder($name).SubFolders for($etr = 0;$etr -lt $folders.count; $etr++) { if($folders[$etr].Name -ne "Forms") { Write-Host "Processing SubFolder ItemCount" -ForegroundColor Red $tempcount = $folders[$etr].ItemCount $subfolderitems += $tempcount } } } } } } } } Write-Host Write-Host Write-Host "Total # of Document Libraries: " $DocLibsCount -ForegroundColor Green Write-Host "Total # of Document Libraries that contain items: " $DocLibwItems -ForegroundColor Green Write-Host "Total # of items: " $totalitems -ForegroundColor Green Write-Host "Total # of items in DocLib\Subfolders: " $subfolderitems -ForegroundColor Green $finishtime = Get-Date Write-Host Write-Host “Script Duration” –ForegroundColor Yellow Write-Host “Started: “ $starttime –ForegroundColor Yellow Write-Host “Finished: “ $finishtime –ForegroundColor Yellow
Stop-SPAssignment -Global
Thanks,