IT/FileServerDedupe
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

automate entire flow

Browse Source
This commit is contained in:
Erik Eckert 2023-09-14 12:06:31 -06:00
parent 273a916f17
commit 9ec984924d
2 changed files with 69 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DuplicatePairing.sql
View File

@ -1,3 +1,5 @@
DROP TABLE OfficeDuplicates;
Create Table OfficeDuplicates AS
SELECT * SELECT *
from FolderData AS Origin from FolderData AS Origin
INNER JOIN FolderData AS Dup ON Dup.Project = Origin.Project INNER JOIN FolderData AS Dup ON Dup.Project = Origin.Project

71
Get-ProjectFolders.ps1
View File

@ -11,12 +11,19 @@ Output File should show:
X:\CG | ProjectNumber | FullPath | filecount X:\CG | ProjectNumber | FullPath | filecount
#> #>
$Refresh = $false #refresh all data - causes the DB to be wiped clean and all file folders to be re-analyzed.
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite" $PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
$OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
$BaseDrive = 'X:\' $BaseDrive = 'X:\'
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}' $ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0 $OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
$OfficeList | ForEach-Object -parallel { $OfficeList | ForEach-Object -parallel {
$Refesh_Copy_Main = $USING:Refresh
$PathToDB_Copy_Main = $USING:PathToDB $PathToDB_Copy_Main = $USING:PathToDB
$ServerProgress = @{ $ServerProgress = @{
ID = 1 ID = 1
@ -35,7 +42,7 @@ $OfficeList | ForEach-Object -parallel {
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
$ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel { $ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel {
# Write-Host "Processing " $_.FullName -ForegroundColor Cyan $Refresh_Thread = $USING:Refresh_Copy_Main
$PathToDB_Copy = $USING:PathToDB_Copy_Main $PathToDB_Copy = $USING:PathToDB_Copy_Main
$JobID = Get-Random $JobID = Get-Random
@ -47,6 +54,17 @@ $OfficeList | ForEach-Object -parallel {
# Write-Progress @CurentLoopProgress # Write-Progress @CurentLoopProgress
if ($Refresh_Thread) {
# Reset the DB, start fresh
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
$ClearQuery = 'DELETE FROM FolderData;'
Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $ClearQuery
}
$projectNumber = '' $projectNumber = ''
# thank you ChatGPT # thank you ChatGPT
$string = $_.FullName $string = $_.FullName
@ -81,7 +99,7 @@ $OfficeList | ForEach-Object -parallel {
$Query = "Select Project FROM FolderData where Project = '$ProjectNumber'" $Query = "Select Project FROM FolderData where Project = '$ProjectNumber'"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
if ($sql_result.project -eq $null) { if (($sql_result.project -eq $null)) {
$CurentLoopProgress = @{ $CurentLoopProgress = @{
ID = $JobID ID = $JobID
Activity = "Loading Telemetry for " + $_.FullName Activity = "Loading Telemetry for " + $_.FullName
@ -107,7 +125,7 @@ $OfficeList | ForEach-Object -parallel {
$out = [PSCustomObject]@{ $out = [PSCustomObject]@{
Server = $_.FullName.Substring(1, 4) -replace (':|\\', '') Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
Project = $ProjectNumber Project = $ProjectNumber.ToString()
Parent = $folder_Parent Parent = $folder_Parent
Path = $_.FullName Path = $_.FullName
FileCount = $folder_FileCount FileCount = $folder_FileCount
@ -125,4 +143,49 @@ $OfficeList | ForEach-Object -parallel {
Write-Progress @CurentLoopProgress -Completed Write-Progress @CurentLoopProgress -Completed
} }
Write-Progress @ServerProgress -Completed Write-Progress @ServerProgress -Completed
} -ThrottleLimit 5 } -ThrottleLimit 5
$PathToDB_Copy_Main = $USING:PathToDB
$ServerProgress = @{
ID = 1
Activity = "Database Work"
PercentComplete = 80
# CurrentOperation = $_.Project
}
Write-Progress @ServerProgress
## Kick off GenTables
$SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$DBConnect = New-SqliteConnection -DataSource $PathToDB
foreach ($office in $OfficeList) {
$OfficeServer = $office.Substring(1, 4) -replace (':|\\', '')
$Query = "select * from OfficeDuplicates where Server = '$OfficeServer'"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) {
$OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
$sql_result | Export-Excel -Path $OutFile -AutoFilter
}
}
# Generate MPE All Xlsx
$Query = "select * from OfficeDuplicates"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) {
$OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
$sql_result | Export-Excel -Path $OutFile -AutoFilter
}
$DBConnect.Close()