migrate to new repo
This commit is contained in:
parent
0971ff8f03
commit
2931268bb3
246
ArchivedCode/Find-DuplicatePaths.ps1
Normal file
246
ArchivedCode/Find-DuplicatePaths.ps1
Normal file
@ -0,0 +1,246 @@
|
||||
<#
|
||||
for each server / Office
|
||||
sanitize project number
|
||||
Compare against each other office
|
||||
if matched
|
||||
get-file counts for each server
|
||||
report duplicate site / count
|
||||
#>
|
||||
|
||||
|
||||
<#
|
||||
Migrating to use a SQLLite DB.
|
||||
|
||||
#>
|
||||
|
||||
|
||||
$PathToDB = "C:\Users\eeckert\Downloads\EgnyteTesting\Dedupe.SQLite"
|
||||
$PathToOutput = "C:\Users\eeckert\Downloads\EgnyteTesting\Output"
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
$CSV_ImportFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun"
|
||||
$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv'
|
||||
|
||||
$Office_to_compare_data = $null
|
||||
foreach ($CSV in $CSV_List) {
|
||||
$csv_temp = Import-Csv $CSV.FullName
|
||||
$Office_to_compare_data += $csv_temp
|
||||
# all Offices loaded into data strcuture 1
|
||||
}
|
||||
|
||||
$Office_to_compare_data = $Office_to_compare_data | sort Project, OfficeServer
|
||||
|
||||
$temp_CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Exclude "Output" | sort name -Descending # get list of CSV's to compare to.
|
||||
$temp_csv_data = $null
|
||||
foreach ($temp_csv in $temp_CSV_List) {
|
||||
|
||||
$temp_csv_temp = Import-Csv -Path $temp_csv.FullName # load comparison data
|
||||
$temp_csv_data += $temp_csv_temp | sort Project, OfficeServer
|
||||
# all offices loaded into comparison data structure
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
$ServerProgress = @{
|
||||
ID = 1
|
||||
Activity = "Processing: " + $CSV.OfficeServer
|
||||
# PercentComplete = 100
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ServerProgress
|
||||
|
||||
$CurrentDate = Get-Date
|
||||
if ($CurrentDate.DayOfWeek -in ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday") -and $CurrentDate.Hour -GT 17 ) {
|
||||
$ThreadLimit = 30
|
||||
}
|
||||
elseif ($CurrentDate.DayOfWeek -in ("Saturday", "Sunday")) {
|
||||
$ThreadLimit = 30
|
||||
}
|
||||
else {
|
||||
$ThreadLimit = 15
|
||||
}
|
||||
|
||||
# foreach ($row in $Office_to_compare_data) {
|
||||
# $Office_to_compare_data | Foreach-Object {
|
||||
$Office_to_compare_data | Foreach-Object -ThrottleLimit $ThreadLimit -Parallel {
|
||||
#Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
|
||||
|
||||
$PSStyle.Progress.View = "Minimal"
|
||||
$DuplicateObj = $null
|
||||
$NonDuplicateObj = $null
|
||||
|
||||
$JobID = Get-Random
|
||||
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Starting: " + $_.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 0
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ComparisonProgress
|
||||
|
||||
# if ($_.Project -in $USING:temp_csv_data.Project) {
|
||||
if ($_.Project -in $USING:temp_csv_data.Project -and $USING:temp_csv_data.OfficeServer -ne $_.OfficeServer) {
|
||||
$ignoreOffice = $_.OfficeServer
|
||||
|
||||
$duplicates = ''
|
||||
$duplicates = $USING:temp_csv_data | where -property Project -eq $_.Project | where -Property OfficeServer -ne $_.OfficeServer
|
||||
# $duplicates = $USING:temp_csv_data | where ((Project -eq $_.Project) -and (OfficeServer -ne $_.OfficeServer))
|
||||
|
||||
if (($_ -eq $duplicates) -or !($duplicates)) {
|
||||
|
||||
}
|
||||
else {
|
||||
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Get Original Data: " + $_.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 15
|
||||
CurrentOperation = "Getting Original data from " + $_.OfficeServer
|
||||
}
|
||||
Write-Progress @ComparisonProgress
|
||||
|
||||
Write-Debug ("Getting file counts from current office...this will take a second...") -Debug
|
||||
# TODO - identify NON-Duplicated files in these file structures...
|
||||
# $row_file_count = [System.IO.Directory]::GetFiles($_.FullPath, "*.*", "AllDirectories") | measure
|
||||
$row_file = get-childitem -path $_.FullPath -recurse -depth 50 -File
|
||||
$row_file_count = $row_file | measure
|
||||
$row_file_lastwrite = $row_file | measure LastWriteTime -Maximum
|
||||
$row_file_lastaccess = $row_file | measure LastAccessTime -Maximum
|
||||
|
||||
foreach ($duplicate in $duplicates) {
|
||||
|
||||
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Get Duplicated data: " + $duplicate.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 33
|
||||
CurrentOperation = "Now getting duplicate path data from " + $duplicate.OfficeServer
|
||||
}
|
||||
Write-Progress @ComparisonProgress
|
||||
|
||||
|
||||
$duplicate_file = get-childitem -path $duplicate.FullPath -recurse -depth 50 -File
|
||||
$duplicate_file_count = $duplicate_file | measure
|
||||
$duplicate_file_lastwrite = $duplicate_file | measure LastWriteTime -Maximum
|
||||
$duplicate_file_lastaccess = $duplicate_file | measure LastAccessTime -Maximum
|
||||
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Processing: " + $duplicate.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 66
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ComparisonProgress
|
||||
|
||||
$result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$ParentPath = $result3.value
|
||||
|
||||
$OriginalProject = "'" + [String]$_.Project
|
||||
$DuplicateProject = "'" + [String]$duplicate.Project
|
||||
|
||||
if ($row_file_count.Count -ge 1 -and $duplicate_file_count.Count -ge 1) {
|
||||
$DuplicateObj = [PSCustomObject]@{
|
||||
OriginalServer = $_.OfficeServer
|
||||
OriginalProject = "'" + $_.project
|
||||
OriginalParent = $ParentPath
|
||||
OriginalPath = $_.FullPath
|
||||
OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0
|
||||
OriginalFileLastWrite = $row_file_Lastwrite.maximum
|
||||
OriginalFileAccess = $row_file_lastaccess.maximum
|
||||
DuplicateServer = $duplicate.OfficeServer
|
||||
DuplicatePath = $duplicate.FullPath
|
||||
DuplicateFileCount = $duplicate_file_count.Count ? $duplicate_file_count.count : 0
|
||||
DuplicateFileLastWrite = $duplicate_file_Lastwrite.maximum
|
||||
DuplicateFileAccess = $duplicate_file_lastaccess.maximum
|
||||
}
|
||||
|
||||
}
|
||||
elseif ($row_file_count.Count -eq 0 -and $duplicate_file_count.Count -gt 0) {
|
||||
# Write-Host "File count is 0 on current side, remote server is source" -ForegroundColor DarkBlue
|
||||
$NonDuplicateObj = [PSCustomObject]@{
|
||||
OriginalServer = $duplicate.OfficeServer
|
||||
OriginalProject = "'" + $duplicate.Project
|
||||
OriginalPath = $duplicate.FullPath
|
||||
OriginalFileCount = $duplicate_file_count.Count ? $duplicate_file_count.Count : 0
|
||||
OriginalFileLastWrite = $duplicate_file_Lastwrite.maximum
|
||||
OriginalFileAccess = $duplicate_file_lastaccess.maximum
|
||||
}
|
||||
}
|
||||
elseif ($row_file_count.count -gt 0 -and $duplicate_file_count.Count -eq 0) {
|
||||
# Write-Host "File count is 0 on one side, not a duplicate..." -ForegroundColor Blue
|
||||
$NonDuplicateObj = [PSCustomObject]@{
|
||||
OriginalServer = $_.OfficeServer
|
||||
OriginalProject = "'" + $_.Project
|
||||
OriginalPath = $_.FullPath
|
||||
OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0
|
||||
OriginalFileLastWrite = $row_file_Lastwrite.maximum
|
||||
OriginalFileAccess = $row_file_lastaccess.maximum
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
elseif ($_.Project -inotin $Office_to_compare_data.Project) {
|
||||
$NonDuplicateObj = [PSCustomObject]@{
|
||||
OriginalServer = $_.OfficeServer
|
||||
OriginalProject = "'" + $_.Project
|
||||
OriginalPath = $_.FullPath
|
||||
OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0
|
||||
OriginalFileLastWrite = $row_file_Lastwrite.maximum
|
||||
OriginalFileAccess = $row_file_lastaccess.maximum
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Writing: " + $_.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 100
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ComparisonProgress -Completed
|
||||
|
||||
|
||||
# Write-Host -ForegroundColor DarkGreen $_.OfficeServer "done, dumping output..."
|
||||
}
|
||||
do {
|
||||
try {
|
||||
|
||||
# $DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result -ErrorAction SilentlyContinue
|
||||
$DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $_.OfficeServer + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
<#Do this if a terminating exception happens#>
|
||||
}
|
||||
|
||||
} while ($Dup_CSV_result.ErrorRecord -ne $null)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
# do {
|
||||
# try {
|
||||
# $NonDuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + 'NonDuplicates.csv') -Append -ErrorVariable Non_Dup_CSV_result -ErrorAction SilentlyContinue
|
||||
|
||||
# }
|
||||
# catch {
|
||||
# <#Do this if a terminating exception happens#>
|
||||
# }
|
||||
|
||||
# } while ($Non_Dup_CSV_Result.ErrorRecord -ne $null)
|
||||
$ComparisonProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Done: " + $_.OfficeServer + ':' + $_.Project
|
||||
PercentComplete = 100
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ComparisonProgress -Completed
|
||||
}
|
||||
|
||||
# $OutPAth = $CSV_ImportFolder + '\Output\' + $CSV.Name + ' Duplicates.csv'
|
||||
|
||||
Write-Progress @ServerProgress -Completed
|
||||
149
ArchivedCode/FindNonDupFiles.ps1
Normal file
149
ArchivedCode/FindNonDupFiles.ps1
Normal file
@ -0,0 +1,149 @@
|
||||
|
||||
|
||||
$CSV_ImportFile = 'M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output'
|
||||
$CSV_ClientList = "M:\IT\Egnyte\DuplicateFiles\Client List.csv"
|
||||
|
||||
$date = Get-Date -Format FileDate
|
||||
$exportPath_main = "M:\IT\Egnyte\DuplicateFiles\$date NonDuplicates.csv"
|
||||
|
||||
$AllDuplicates_files = Get-ChildItem -Path $CSV_ImportFile -Filter "* Duplicates.csv" -File
|
||||
$AllDuplicates = @()
|
||||
foreach ($file in $AllDuplicates_files) {
|
||||
$temp_val = Import-Csv $file.FullName
|
||||
$AllDuplicates += $temp_val
|
||||
}
|
||||
|
||||
$ClientList = Import-csv $CSV_ClientList
|
||||
|
||||
$BaseDrive = 'X:\'
|
||||
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
|
||||
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
|
||||
|
||||
$OfficeList | ForEach-Object -ThrottleLimit 3 -Parallel {
|
||||
#Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
|
||||
|
||||
$XDrivePath = $_.FullName
|
||||
Write-Host "Processing " $_.FullName -ForegroundColor Green
|
||||
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
|
||||
|
||||
$All_DupParent_Main = $USING:AllDuplicates.OriginalParent | sort -Unique
|
||||
$All_DupProject_Main = $USING:AllDuplicates.OriginalProject | sort -Unique
|
||||
$exportPath = $USING:exportPath_main
|
||||
|
||||
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
|
||||
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
|
||||
|
||||
$ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel { #uncomment for threaded mode
|
||||
# $ProjectFolders | ForEach-Object { #comment for threaded mode
|
||||
Write-Host "Processing " $_.FullName -ForegroundColor Cyan
|
||||
$projectNumber = ''
|
||||
# thank you ChatGPT
|
||||
$string = $_.FullName
|
||||
$regex = "\\(\d+(-\d+)*)"
|
||||
$matches = [regex]::Matches($string, $regex)
|
||||
$result = ""
|
||||
foreach ($match in $matches) {
|
||||
$result += $match.Groups[1].Value
|
||||
if ($match.Groups[2].Success) {
|
||||
$result += $match.Groups[2].Value
|
||||
}
|
||||
# if ($match.Groups[3].Success) {
|
||||
# $result += $match.Groups[3].Value
|
||||
# }
|
||||
}
|
||||
|
||||
$projectnumber = ("'" + $result)
|
||||
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$ParentPath = $result3.value
|
||||
|
||||
# / Thank you Chat GPT
|
||||
if ($projectnumber.Length -lt 6 ) {
|
||||
# write-host $_.FullName " is not a project" -ForegroundColor Cyan
|
||||
}
|
||||
elseif ($projectnumber.Length -ge 6) {
|
||||
$out = $null
|
||||
$OfficeRegex = "(X:\\[A-Z]+)"
|
||||
$str = $_.FullName -replace ($OfficeRegex, "")
|
||||
$OfficeServer = $_.FullName.Substring(1, 4) -replace (':|\\', '')
|
||||
|
||||
$All_DupParent_copy = $USING:All_DupParent_main
|
||||
|
||||
|
||||
if ($ParentPath -notin $All_DupParent_copy) {
|
||||
$str = $ParentPath -replace ($OfficeRegex, "")
|
||||
$EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/")
|
||||
|
||||
$fileList = Get-ChildItem $_.FullName -Recurse -Depth 1
|
||||
$FileSize = $fileList | measure -Sum Length
|
||||
$totalFiles = $fileList | measure
|
||||
$LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum
|
||||
$LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum
|
||||
|
||||
|
||||
|
||||
$out = [PSCustomObject]@{
|
||||
OfficeServer = $OfficeServer
|
||||
ProjectNumber = $projectNumber
|
||||
ParentPath = $ParentPath
|
||||
UNCPath = $ParentPath -replace ('X:', "\\mpe.ca\datadrive")
|
||||
EgnytePath1 = $EgnytePath1
|
||||
# EgnytePath2 = $EgnytePath2
|
||||
FileCount = $totalFiles.Count
|
||||
FolderSize = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB
|
||||
LastWrite = $LastWriteDate.Maximum
|
||||
LastAccess = $LastAccessDate.Maximum
|
||||
Parent = "Yes"
|
||||
}
|
||||
do {
|
||||
try {
|
||||
|
||||
$out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
<#Do this if a terminating exception happens#>
|
||||
}
|
||||
|
||||
} while ($CSV_result.ErrorRecord -ne $null)
|
||||
}
|
||||
elseif ($projectNumber -notin $USING:All_DupProject_Main) {
|
||||
$str = $_.FullName -replace ($OfficeRegex, "")
|
||||
$EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/")
|
||||
|
||||
$fileList = Get-ChildItem $_.FullName -Recurse -Depth 50
|
||||
$FileSize = $fileList | measure -Sum Length
|
||||
$totalFiles = $fileList | measure
|
||||
$LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum
|
||||
$LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum
|
||||
|
||||
if (($FileSize.Sum -gt 0) -or ($totalFiles.Count -gt 0)) {
|
||||
# no need to do anything if either value is Zero.
|
||||
$out = [PSCustomObject]@{
|
||||
OfficeServer = $OfficeServer
|
||||
ProjectNumber = $projectNumber
|
||||
ParentPath = $ParentPath
|
||||
UNCPath = $_.FullName -replace ('X:', "\\mpe.ca\datadrive")
|
||||
EgnytePath1 = $EgnytePath1
|
||||
# EgnytePath2 = $EgnytePath2
|
||||
FileCount = $totalFiles.Count
|
||||
FolderSize = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB
|
||||
LastWrite = $LastWriteDate.Maximum
|
||||
LastAccess = $LastAccessDate.Maximum
|
||||
Parent = "No"
|
||||
}
|
||||
}
|
||||
do {
|
||||
try {
|
||||
|
||||
$out | Export-Csv -Path $USING:exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
|
||||
# $out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
<#Do this if a terminating exception happens#>
|
||||
}
|
||||
|
||||
} while ($CSV_result.ErrorRecord -ne $null)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
38
ArchivedCode/ParentSquish.ps1
Normal file
38
ArchivedCode/ParentSquish.ps1
Normal file
@ -0,0 +1,38 @@
|
||||
<#
|
||||
Import all NonDuplicates.
|
||||
Find "root" path for each client folder.
|
||||
if a ROOT folder appears in the Duplicate list (MPE All.CSV), then we can't migrate.
|
||||
If a root does not appear, we can output that to...something...and these can be added to the migration list without subfolders.
|
||||
|
||||
#>
|
||||
|
||||
|
||||
$date = Get-Date -Format FileDate
|
||||
$ImportPath = "M:\IT\Egnyte\DuplicateFiles\20230906 NonDuplicates.csv"
|
||||
$DuplicateImportPath = "M:\IT\Egnyte\DuplicateFiles\MPE All.csv"
|
||||
|
||||
$Non_Duplicate_Folders = Import-Csv -Path $ImportPath
|
||||
$Duplicate_Folders = Import-Csv -Path $DuplicateImportPath
|
||||
|
||||
$dupfolders = @()
|
||||
foreach ($Dup_Folder in $Duplicate_Folders) {
|
||||
$Duplicateresult = [Regex]::Matches($Dup_Folder.OriginalPath.Replace('X:\', '\\mpe.ca\datadrive\'), "^(.*[\\\/])")
|
||||
$dupfolders += $Duplicateresult.Value
|
||||
|
||||
|
||||
}
|
||||
$dupfolders = $dupfolders | sort -Unique
|
||||
$folder = [PSCustomObject]@()
|
||||
$Non_Duplicate_Folders | foreach {
|
||||
|
||||
|
||||
$result3 = [Regex]::Matches($_.UNCPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$ParentPath = $result3.value
|
||||
$out = @()
|
||||
if ($ParentPath -notin $dupfolders) {
|
||||
$out += [PSCustomObject]@{
|
||||
ParentPath = $ParentPath
|
||||
}
|
||||
}
|
||||
$folder += $out
|
||||
}
|
||||
127
DataBaseLoad.ps1
Normal file
127
DataBaseLoad.ps1
Normal file
@ -0,0 +1,127 @@
|
||||
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
|
||||
$PathToOutput = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
|
||||
$Refresh = $false
|
||||
|
||||
|
||||
$DBExist = Test-Path $PathToDB
|
||||
|
||||
if (!($DBExist)) {
|
||||
# Create SQLite DB
|
||||
$Query = "Create Table FolderData (Server TEXT, Project TEXT, Parent TEXT, Path TEXT,FileCount INTEGER,FileSize REAL, FileLastWrite DATETIME, FileLastAccess DATETIME);PRAGMA journal_mode=WAL;"
|
||||
$QResult = Invoke-SqliteQuery -Database $PathToDB -Query $Query
|
||||
|
||||
|
||||
}
|
||||
|
||||
$CSV_ImportFolder = "C:\Users\eeckert\Downloads\EgnyteTesting"
|
||||
$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv'
|
||||
|
||||
$Office_to_compare_data = $null
|
||||
foreach ($CSV in $CSV_List) {
|
||||
$csv_temp = Import-Csv $CSV.FullName
|
||||
$Office_to_compare_data += $csv_temp
|
||||
# all Offices loaded into data strcuture 1
|
||||
}
|
||||
|
||||
# Load Data into SQL DB
|
||||
$Office_to_compare_data = $Office_to_compare_data | sort -Property Project, OfficeServer
|
||||
$Office_to_compare_data | ForEach-Object -ThrottleLimit 15 -parallel { #-ThrottleLimit 5 -parallel
|
||||
|
||||
$ErrorLog = $USING:PathToOutput + '\SQL Error Log.txt'
|
||||
$DBCopy = $USING:PathToDB #copy DB Path into local runspace
|
||||
$Refresh_Copy = $USING:Refresh
|
||||
# $ErrorLog = $PathToOutput + '\SQL Error Log.txt'
|
||||
# $DBCopy = $PathToDB #copy DB Path into local runspace
|
||||
# $Refresh_Copy = $Refresh
|
||||
|
||||
$JobID = Get-Random
|
||||
$ProgressData = @{
|
||||
ID = $JobID
|
||||
Activity = "Analysing:" + $_.OfficeServer + ": " + $_.Project
|
||||
PercentComplete = 10
|
||||
|
||||
}
|
||||
Write-Progress @ProgressData
|
||||
|
||||
$OfficeServer = $_.OfficeServer
|
||||
$Project = $_.Project
|
||||
$FullPath = $_.FullPath
|
||||
|
||||
$DBConn = New-SQLiteConnection -Database $DBCopy
|
||||
|
||||
$Query = "SELECT Project FROM FolderData where Project = '$Project';"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $DBCopy -Query $Query
|
||||
|
||||
if ($sql_result.Project -eq $null -or $Refresh_Copy -eq $true) {
|
||||
# No need to re-do what we've done - move on to the next one
|
||||
|
||||
if ($DBConn.State -eq "Open") {
|
||||
|
||||
$result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
|
||||
$ProgressData = @{
|
||||
ID = $JobID
|
||||
Activity = "Getting file data for:" + $_.OfficeServer + ": " + $_.Project
|
||||
PercentComplete = 25
|
||||
|
||||
}
|
||||
Write-Progress @ProgressData
|
||||
|
||||
$FolderPath_Data = Get-ChildItem -Path $_.FullPath -File -Depth 50 -Recurse
|
||||
$folder_Parent = $result3.value
|
||||
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
|
||||
$folder_FileCount = ($FolderPath_Data | measure).Count
|
||||
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
|
||||
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
|
||||
|
||||
$ProgressData = @{
|
||||
ID = $JobID
|
||||
Activity = "Writing to DB:" + $_.OfficeServer + ": " + $_.Project
|
||||
PercentComplete = 75
|
||||
|
||||
}
|
||||
Write-Progress @ProgressData
|
||||
|
||||
|
||||
## Build Query and hopefully write to DB
|
||||
|
||||
if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) {
|
||||
#Let's ignore Null Values
|
||||
}
|
||||
else {
|
||||
|
||||
try {
|
||||
$Query = "Insert into FolderData (Server,Project,Parent,Path,FileCount,FileSize,FileLastWrite,FileLastAccess) VALUES ('$OfficeServer','$Project',`"$folder_Parent`",`"$FullPath`",$folder_FileCount,$folder_FileSize,'$folder_LastWrite','$Folder_LastAccess')"
|
||||
Invoke-SqliteQuery -Database $DBCopy -Query $Query #-ErrorVariable SQLError
|
||||
}
|
||||
catch {
|
||||
$timestamp = get-date
|
||||
$errorMessage = $SQLError.ErrorRecord
|
||||
|
||||
$OutString = "[$timestamp] `t $errorMessage `t Query: `t $Query"
|
||||
|
||||
try {
|
||||
$OutString | Add-Content -Path $ErrorLog
|
||||
}
|
||||
catch {
|
||||
New-Item -Path $ErrorLog -Value $OutString
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
$DBConn.Close()
|
||||
$ProgressData = @{
|
||||
ID = $JobID
|
||||
Activity = "Analysing:" + $_.OfficeServer + ": " + $_.Project
|
||||
PercentComplete = 100
|
||||
}
|
||||
Write-Progress @ProgressData -Completed
|
||||
|
||||
}
|
||||
|
||||
128
Get-ProjectFolders.ps1
Normal file
128
Get-ProjectFolders.ps1
Normal file
@ -0,0 +1,128 @@
|
||||
<#
|
||||
Script should:
|
||||
1) get list of all Project Folders
|
||||
2) Determine if there's content in these folders (exlude shortcuts, lnks, single files)
|
||||
3) Flag folders with content
|
||||
4) Output results to file for each server.
|
||||
|
||||
Each Output file should be read by the analysis script to determine locations with duplicate data.
|
||||
|
||||
Output File should show:
|
||||
X:\CG | ProjectNumber | FullPath | filecount
|
||||
|
||||
#>
|
||||
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
|
||||
$BaseDrive = 'X:\'
|
||||
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
|
||||
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
|
||||
|
||||
$OfficeList | ForEach-Object -parallel {
|
||||
$PathToDB_Copy_Main = $USING:PathToDB
|
||||
$ServerProgress = @{
|
||||
ID = 1
|
||||
Activity = "Processing: " + $_.FullName
|
||||
# PercentComplete = 100
|
||||
# CurrentOperation = $_.Project
|
||||
}
|
||||
Write-Progress @ServerProgress
|
||||
#Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
|
||||
|
||||
$XDrivePath = $_.FullName
|
||||
# Write-Host "Processing " $_.FullName -ForegroundColor Green
|
||||
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
|
||||
|
||||
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
|
||||
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
|
||||
|
||||
$ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel {
|
||||
# Write-Host "Processing " $_.FullName -ForegroundColor Cyan
|
||||
$PathToDB_Copy = $USING:PathToDB_Copy_Main
|
||||
|
||||
$JobID = Get-Random
|
||||
$CurentLoopProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Analyzing " + $_.FullName
|
||||
PercentComplete = 10
|
||||
}
|
||||
|
||||
Write-Progress @CurentLoopProgress
|
||||
|
||||
$projectNumber = ''
|
||||
# thank you ChatGPT
|
||||
$string = $_.FullName
|
||||
$regex = "\\(\d+(-\d+)*)"
|
||||
$matches = [regex]::Matches($string, $regex)
|
||||
$result = ""
|
||||
foreach ($match in $matches) {
|
||||
$result += $match.Groups[1].Value
|
||||
if ($match.Groups[2].Success) {
|
||||
$result += $match.Groups[2].Value
|
||||
}
|
||||
# if ($match.Groups[3].Success) {
|
||||
# $result += $match.Groups[3].Value
|
||||
# }
|
||||
}
|
||||
# / Thank you Chat GPT
|
||||
# $result = $result -replace ('-','')
|
||||
|
||||
|
||||
if ($result.Length -lt 5 ) {
|
||||
# $CurentLoopProgress = @{
|
||||
# ID = $JobID
|
||||
# Activity = $_.FullName + " Not a Project"
|
||||
# }
|
||||
|
||||
# Write-Progress @CurentLoopProgress
|
||||
}
|
||||
elseif ($result.Length -ge 5) {
|
||||
|
||||
$projectnumber = $result
|
||||
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
|
||||
$Query = "Select Project FROM FolderData where Project = '$ProjectNumber'"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
|
||||
|
||||
if ($sql_result.project -eq $null) {
|
||||
$CurentLoopProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Loading Telemetry for " + $_.FullName
|
||||
PercentComplete = 75
|
||||
}
|
||||
|
||||
Write-Progress @CurentLoopProgress
|
||||
|
||||
|
||||
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
|
||||
$folder_Parent = $result3.value
|
||||
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
|
||||
$folder_FileCount = ($FolderPath_Data | measure).Count
|
||||
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
|
||||
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
|
||||
|
||||
if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) {
|
||||
|
||||
#Let's ignore Null Values
|
||||
}
|
||||
else {
|
||||
|
||||
$out = [PSCustomObject]@{
|
||||
Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
|
||||
Project = $ProjectNumber
|
||||
Parent = $folder_Parent
|
||||
Path = $_.FullName
|
||||
FileCount = $folder_FileCount
|
||||
FileSize = $folder_FileSize
|
||||
FileLastWrite = $folder_LastWrite
|
||||
FileLastAccess = $folder_LastAccess
|
||||
} | Out-DataTable
|
||||
|
||||
Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
|
||||
|
||||
}
|
||||
}
|
||||
$DBConnect.Close()
|
||||
}
|
||||
Write-Progress @CurentLoopProgress -Completed
|
||||
}
|
||||
Write-Progress @ServerProgress -Completed
|
||||
} -ThrottleLimit 5
|
||||
Loading…
Reference in New Issue
Block a user