Using powershell to check company information

I had a long list of companynames but no homepage adress.
Using the swedish yellow pages service Eniro I could use powershell to get the information.
This script only gets the first result with homepage attribut added, but depending on what you put the results may vary.

	function ping-company {
	param(
	$profile = "<profileName>",
	$key = "<apiKey>",
	$version = "1.1.3",
	$searchWord
	)
	$url = "https://api.eniro.com/cs/search/basic?profile=$($profile)&key=$($key)&country=se&version=$($version)&search_word=$($searchWord)"
	$result = Invoke-RestMethod -Uri $url
	$queryResult = $result.adverts |where-object {![string]::IsNullOrEmpty($_.homepage)} |select -first 1
	$request = Invoke-WebRequest -Uri $queryResult.homepage
	$homepage = (($($request.ParsedHtml.getElementsByTagName("meta")|select -ExpandProperty content)).split(";")|where-object {$_ -like "url=*"}).replace("url=","")
	return $homepage
	}
	$searchWord = "Lantmännen"
	$searchWord = [System.Web.HttpUtility]::UrlEncode($searchWord)
	ping-company -searchWord $searchWord

view raw getCompanyhomepage hosted with ❤ by GitHub

Checkup list of websites

To determine a range of websites dns, registrar and IPOwner and what technology is used the following script can be used.
Note that whois64.exe referred to in script needs to be downloaded seperatly from Sysinternals/Microsoft

	$scriptPath = $(split-path -Parent $myinvocation.MyCommand.Definition)
	#settings
	$outputFilePath = "$($scriptPath)\WebsiteList_cms.csv"
	#settings end
	function ping-cms {
	param(
	$address
	)
	$apiKey = <insertYourAPIKey>
	$baseUrl = "https://whatcms.org/APIEndpoint/Detect?key=$apiKey&url="
	$url = "$($baseUrl)$address"
	$result = Invoke-RestMethod -Uri $url
	$object = $result.result
	$object |add-member -MemberType NoteProperty -Name "Url" -Value $address -Force
	return $object
	}
	function ping-cmsdetails {
	param(
	$address
	)
	$apiKey = <insertYourAPIKey>
	$baseUrl = "https://whatcms.org/APIEndpoint/Technology?key=$apiKey&url="
	$url = "$($baseUrl)$address"
	$result = Invoke-RestMethod -Uri $url
	$object = new-object psobject -Property @{Url=$address}
	foreach ($subitem in $result.results) {
	$object |Add-Member -MemberType NoteProperty -Name $subitem.categories[0] -Value $subitem.name -Force
	}
	return $object
	}
	$startTime = get-date
	write-output "script started at $startTime"
	$scriptPath = $(split-path -parent $myinvocation.mycommand.Definition)
	$collection = import-csv -path "$($scriptpath)\siteList.csv" -Delimiter ";" -Encoding UTF8 #|select -first 2
	$loops = 1
	$allObjects = @()
	foreach ($item in $collection) {
	$formattedName = $item.site.trimstart("https://")
	$formattedName = $formattedName.trimstart("http://")
	$formattedName = $formattedName.split("/")[0]
	#$allObjects += ping-cms -address $formattedName
	$allObjects += ping-cmsdetails -address $formattedName
	start-sleep -Seconds 12
	write-output "$($loops): Sleeping... for item $($loops) out of $($collection.count) items"
	$loops++
	}
	$allObjects | export-csv -Path $outputFilePath -Encoding UTF8 -NoTypeInformation
	$endtime = get-date
	write-output "Done collecting $([math]::round((($endtime-$starttime).totalseconds),2)) seconds"

view raw scanCMS.ps1 hosted with ❤ by GitHub

	$scriptPath = $(split-path -parent $MyInvocation.MyCommand.Definition)
	$inputFilePath = "$($scriptPath)\SiteList.csv"
	$outputFilePath = "$($scriptPath)\exportedWhoIs.csv"
	$collection = import-csv -Path $inputFilePath -Delimiter ";" -Encoding UTF8 #|select -First 12
	Import-Module "$($scriptPath)\whoisTools.psm1" -force
	Remove-Variable object -ErrorAction SilentlyContinue
	Push-Location $scriptPath
	.\whois64.exe -accepteula |out-null
	$object = @()
	$count = 0
	foreach ($item in $collection) {
	Remove-Variable properties,tempVar -ErrorAction SilentlyContinue
	$tempVar = $item.Site.TrimStart("https://")
	$tempvar = $tempVar.TrimStart("http://")
	$formattedName = $tempVar.Split("/")[0].trimstart("www.")
	$count++
	write-output "$($count): Running whois on $formattedName..."
	#check if åäö
	if ($formattedName -match '[å-öÅ-Ö]') {
	$formattedNameUnicode = convert-idn -domain $formattedName -coding ascii
	$whoisoutput = .\whois64.exe -nobanner -v $formattedNameUnicode
	}
	else {
	$whoisoutput = .\whois64.exe -nobanner -v $formattedName
	}
	$tempObject = get-whoisvalues -whoisinput $whoisoutput -formattedName $formattedName
	$object += $tempObject
	write-output "Done with $formattedName"
	}
	$object |export-csv -Path $outputFilePath -Encoding UTF8 -NoTypeInformation
	Pop-Location
	$object

view raw whois2.ps1 hosted with ❤ by GitHub

	function convert-idn($domain,$coding) {
	$idn = new-object System.Globalization.IdnMapping
	if ($coding -eq "ascii") {
	$idn.GetAscii($domain)
	}
	elseif ($coding -eq "unicode") {
	$idn.GetUnicode($domain)
	}
	}
	function get-whoisvalues {
	param(
	[CmdLetBinding()]
	[parameter(mandatory=$true)]
	[validatenotnullorempty()]
	$whoisinput,
	$formattedName
	)
	begin {
	remove-variable properties -ErrorAction SilentlyContinue
	}
	process {
	$properties = @{site=$formattedName}
	#$properties = @{}
	Write-Verbose -Message "Starting verbose..."
	$newNsLoop = 1
	write-verbose -Message "Input is: $($input.count) lines"
	write-verbose -Message "Input: $($input)"
	write-verbose -Message "Input: $($input.gettype())"
	foreach ($line in $whoisinput) {
	if ($line -like "state:*") {
	$properties.state = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line -like "domain:*" -or $line -like "Domain Name:*") {
	$properties.domain = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line -like "holder:*" -or $line -like "Registrant Organization:*") {
	$properties.holder = $line.split(":")[-1].trimstart(" ")
	}
	elseif ( $line -like "created:*" -or $line -like "Creation Date:*") {
	$properties.created = $line.split(":")[1].trimstart(" ")
	}
	elseif ($line -like "modified:*" -or $line -like "Updated Date:*") {
	$properties.modified = $line.split(":")[1].trimstart(" ")
	}
	elseif ($line -like "expires:*" -or $line -like "Registrar Registration Expiration Date:*" -or $line -like "Registry Expiry Date:*") {
	$properties.expires = $line.split(":")[1].trimstart(" ")
	}
	elseif ($line -like "nserver:*" -or $line -like "Name Server:*") {
	if ($newNsLoop -eq 1) {
	$properties.nsserver1 = $line.split(":")[-1].trimstart(" ")
	$newNsLoop++
	}
	elseif ($newNsLoop -eq 2) {
	$properties.nsserver2 = $line.split(":")[-1].trimstart(" ")
	$newNsLoop++
	}
	elseif ($newNsLoop -eq 3) {
	$properties.nnsserver3 = $line.split(":")[-1].trimstart(" ")
	}
	else { write-output "Couldnt match ns"}
	}
	elseif ($line -like "status:*" -or $line -like "Domain Status:") {
	$properties.status = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line -like "transferred:*") {
	$properties.transferred = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line.ToLower() -like "dnssec:*") {
	$properties.dnssec = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line.ToLower() -like "registrar whois server:*") {
	$properties.whois = $line.split(":")[-1].trimstart(" ")
	}
	elseif ($line.ToLower() -like "registrar:*") {
	$properties.registrar = $line.split(":")[-1].trimstart(" ")
	}
	}#end foreach
	#$properties.site=$formattedName
	Write-Verbose "Properties has $($properties.count)"
	$tempObject = new-object psobject -Property $properties
	return $tempObject
	}
	}
	function ping-ripe {
	param(
	$baseUrl = 'http://rest.db.ripe.net/search.json?query-string=',
	$ip = '81.170.175.179'
	)
	$url = "$($baseUrl)$($ip)"
	$result = Invoke-RestMethod -uri $url
	$values = $result.objects.object.attributes |select @{name='Attribute';expression={$_.attribute|where-object {$_.name -like 'descr'}}}
	$outputObject = $attributeObject = @()
	foreach ($value in $values) {
	$tempObject = new-object psobject -Property @{value=$value.attribute.value}
	$attributeObject += $tempObject
	}
	return $outputObject += new-object psobject -Property @{input=$ip;value=$($attributeObject.value -join ";")}
	}
	function ping-arin {
	param(
	$baseUrl = "http://whois.arin.net/rest/ip/",
	$ip
	)
	$url = "$($baseUrl)$ip"
	$headers = @{"Accept"="application/json"}
	$result = Invoke-RestMethod -uri $url -Headers $headers
	$outputObject = new-object psobject -Property @{input=$ip;value=$result.net.orgRef.'@name'}
	return $outputObject
	}

view raw whoIsTools.psm1 hosted with ❤ by GitHub

	$scriptPath = $(split-path -Parent -Path $myinvocation.MyCommand.Definition)
	Import-Module "$($scriptPath)\whoisTools.psm1" -force
	#settings
	$inputFilePath = "$($scriptPath)\siteList.csv"
	$outputFilePath = "$($scriptPath)\outputIPOwners.csv"
	#settings end
	$collection = import-csv -Path $inputFilePath -Delimiter ";" |select -ExpandProperty Site
	$outputObjects = @()
	#get ip for address
	foreach ($item in $collection) {
	remove-variable testConnection,round -ErrorAction SilentlyContinue
	write-output "Checking $item"
	if ($item -like "http://*") {
	$address = $item.Replace("http://","").trimEnd("/")
	}
	elseif ($item -like "https://*") {
	$address = $item.replace("https://","").trimEnd("/")
	}
	else {
	write-warning "$item couldn't find anything to remove"
	$address = $item.TrimEnd("/")
	}
	$Address = $address.Split("/")[0]
	$testConnection = [System.Net.Dns]::GetHostAddresses($address)[0]
	if (!([String]::IsNullOrEmpty($testconnection))) {
	write-output "$address : got ip $($testconnection.ipaddresstostring)"
	$round = new-object psobject -Property @{hostName=$address;ip=$testConnection.ipaddresstostring}
	}
	else {
	write-output "$address : unresolved "
	$round = new-object psobject -Property @{hostName='unresolved'}
	}
	if ($round.hostName -notlike "unresolved") {
	$answer = ping-arin -ip $round.ip
	if ($answer.value -like "*RIPE*") {
	$answer = ping-ripe -ip $round.ip
	}
	$answer |Add-Member -MemberType NoteProperty -Name address -Value $address -Force
	$outputObjects += $answer
	}
	elseif ($round.hostName -like "unresolved") {
	write-output "$address is unresolved"
	$unAnswered = new-object psobject -Property @{value=$round.hostName;address=$address}
	$outputObjects += $unAnswered
	}
	}
	$outputObjects #|sort-object address |select address,value
	$outputObjects |select input,value,address|export-csv -Path $outputFilePath -Encoding UTF8 -NoTypeInformation

view raw whoOwnIP2.ps1 hosted with ❤ by GitHub

References:
https://docs.microsoft.com/en-us/sysinternals/downloads/whois

enumerate columns from subsites

This case is something which I've came back to a few times.
Customer wants to output columns from certain content types from a number of sites.
Its always a hassle to get the correct names of columns so I've built a method to output the values more dynamically.

1. First getFields.ps1 reads columns containg values from a list item, outputs to gridview, chosen values gets saved to customfields.csv
2. Second getSitecontenttypefields.ps1 enumerates all sites on chosen destination and loops all items of specified content types and outputs chosen fields from customfields.ps1 to an object which is saved to a csv-file.

	$customUrl = "https://mysiteadress.com/mysubsite/publications/2018/Pages/2018-21.aspx"
	Add-PSSnapin microsoft.sharepoint.powershell
	$scriptPath = split-path -Parent $myinvocation.MyCommand.Definition
	Remove-Variable -ErrorAction SilentlyContinue savedFields
	$webUrl = $customUrl.replace($customUrl.split("/")[-1],"").replace($customUrl.split("/")[-2],"").TrimEnd("/")
	$web = Get-SPWeb $webUrl
	$object = @()
	$oneItem = $web.GetListItem($url)
	foreach ($field in $oneItem.ContentType.fields) {
	if (![string]::IsNullOrEmpty($oneItem[$field.staticName])) {
	$object += new-object psobject -Property @{fieldName=$field.StaticName;fieldValue=$oneItem[$field.staticName]}
	}
	}
	write-output "Found $($object.count) items"
	$savedFields = $object |select fieldName,fieldValue |Out-GridView -Title "Choose columns" -PassThru
	if (![string]::IsNullOrEmpty($savedFields)) {
	$savedFields |select fieldname | export-csv -Path "$($scriptPath)\customfieldsList.csv" -Encoding UTF8 -NoTypeInformation
	write-output "Output $(($savedFields|measure-object).count) fields to file"
	}
	else {
	write-output "No fields chosen"
	}

view raw getFields.ps1 hosted with ❤ by GitHub

	$siteCollection = "https://mysiteadress.com"
	$webRootUrl = "https://mysiteadress.com/mysubsite/publications/"
	$contentTypeName = "my-content-type"
	#$VerbosePreference = "continue" #uncomment to show verbose
	$scriptPath = split-path -Parent $myinvocation.MyCommand.Definition
	import-module "$($scriptPath)\SpSiteTools.psm1" -force
	$fieldNames = import-csv -path "$($scriptPath)\customfieldsList.csv"
	Add-PSSnapin microsoft.sharepoint.powershell
	$webSites = get-SubSites -siteToCountWebSites $webRootUrl -siteCollection $siteCollection
	$outputObjects = get-columnvalues -websites $websites -contenttype $contentTypeName -fields $fieldNames -Verbose
	write-output "Found $($outputObjects.count) items in $($website.count) websites"
	$outputObjects |export-csv -Path "$($scriptPath)\output2.csv" -NoTypeInformation -Encoding UTF8

view raw getSiteContentTypeFields.ps1 hosted with ❤ by GitHub

	function get-SubSites ($siteToCountWebSites,$siteCollection)
	{
	$rootSite = get-spweb -Identity $siteToCountWebSites
	$countSites= get-spweb -Site $siteCollection -limit all |where-object {$_.url -like "$($rootSite.url)*"}
	return $countSites
	}
	function select-tag {
	param($string,
	$tag="a")
	$htmlString2 = new-object -ComObject "HTMLFILE"
	$htmlString2.IHTMLDocument2_write($string)
	$htmlString2.body.getElementsByTagName($tag)|select -ExpandProperty pathname
	}
	function get-pageLib {
	param(
	$website
	)
	if ($website.lists.title -contains 'Sidor') {
	$listName = "Sidor"
	}
	elseif ($website.lists.title -contains 'Pages') {
	$listName = "Pages"
	}
	else {
	write-verbose "Couldn't find list!"
	$listname = $null
	}
	return $listName
	}
	function get-columnvalues {
	param(
	[parameter(mandatory=$true)]
	[validateNotNullOrEmpty()]
	$websites,
	[parameter(mandatory=$true)]
	[validateNotNullOrEmpty()]
	$contentType,
	[parameter(mandatory=$true)]
	[validateNotNullOrEmpty()]
	$fields
	)
	$objects = @()
	foreach ($website in $webSites) {
	$listName = get-pageLib -website $website
	write-verbose "Using library: $listName"
	$listItems = $website.lists[$listName].items
	foreach ($item in $listItems) {
	if ($item.ContentType.Name -like $contentTypeName) {
	write-verbose "Content Type: $($item.contentType.name)"
	$properties = @{}
	foreach ($fieldName in $fieldNames.fieldName) {
	$properties.add($fieldName,$item[$fieldName])
	}
	$objects += new-object psobject -Property $properties
	}
	else {
	write-verbose "Content Type: $($item.contentType.name)"
	}
	}
	}
	return $objects
	}

view raw SPSiteTools.psm1 hosted with ❤ by GitHub