IT/FileServerDedupe
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'New_strat' (#1) from New_strat into main

Browse Source 
Reviewed-on: #1
This commit is contained in:
Erik Eckert 2023-10-04 16:00:34 -06:00
commit 187cd43550
3 changed files with 146 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Generate Egnyte Export-old.sql Normal file
View File

@ -0,0 +1,24 @@
-- Generate Non-Duplicated PARENT FOLDER List
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS
Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
Where Parent in (
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
)
union
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;

26
Generate Egnyte Export.sql
View File

@ -1,24 +1,14 @@
-- Generate Non-Duplicated PARENT FOLDER List DROP TABLE IF EXISTS DataInventory;
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS create table DataInventory as
Select * from ( Select * from (
SELECT SERVER, SELECT SERVER,
REPLACE( REPLACE(
Parent, Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath, "X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath, REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status from FolderData
Where Parent in ( WHERE FolderData.fld_MigrationID IS NULL
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
GROUP by UNCPath GROUP by UNCPath
) ORDER by UNCPath
union );
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;

238
Get-ProjectFolders.ps1
View File

@ -27,7 +27,7 @@ $PAth_To_Migration_Status = "M:\00 - Egnyte PS Data Inventory\Migration_status\M
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite" $PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
$OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output" $OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
$BaseDrive = 'X:\' $BaseDrive = 'X:\'
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}' $ProjectRegex = '^X:\\[A-Z]{2}\W[a-zA-Z0-9 ]*\\'
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0 $OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
if ($Refresh) { if ($Refresh) {
@ -55,7 +55,7 @@ $OfficeList | ForEach-Object -parallel {
$XDrivePath = $_.FullName $XDrivePath = $_.FullName
# Write-Host "Processing " $_.FullName -ForegroundColor Green # Write-Host "Processing " $_.FullName -ForegroundColor Green
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex $ProjectFolders = get-childitem $_.FullName -Directory -Depth 1 | Where-Object FullName -Match $USING:ProjectRegex
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
@ -73,97 +73,61 @@ $OfficeList | ForEach-Object -parallel {
# Write-Progress @CurentLoopProgress # Write-Progress @CurentLoopProgress
$projectNumber = ''
# thank you ChatGPT # thank you ChatGPT
$string = $_.FullName $string = $_.FullName
$regex = "\\(\d+(-\d+)*)" $RefreshDays = -15 - (Get-Random -Maximum 5)
$matches = [regex]::Matches($string, $regex) $Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
$result = "" $Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path
foreach ($match in $matches) { $Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
$result += $match.Groups[1].Value $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
if ($match.Groups[2].Success) { $Query = "Select fld_LastRefresh FROM FolderData where Server = '$Server' AND Path = `"$Path`";"
$result += $match.Groups[2].Value $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
if (($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
$CurentLoopProgress = @{
ID = $JobID
Activity = "Loading Telemetry for " + $_.FullName
PercentComplete = 75
} }
# if ($match.Groups[3].Success) {
# $result += $match.Groups[3].Value
# }
}
# / Thank you Chat GPT
# $result = $result -replace ('-','')
if ($result.Length -lt 5 ) {
# $CurentLoopProgress = @{
# ID = $JobID
# Activity = $_.FullName + " Not a Project"
# }
# Write-Progress @CurentLoopProgress
}
elseif ($result.Length -ge 5) {
$RefreshDays = -15 - (Get-Random -Maximum 5)
$Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
$Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path
$Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
$projectnumber = $result.tostring()
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
$Query = "Select Project,fld_LastRefresh FROM FolderData where Project = '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`";"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
if (($sql_result.project -eq $null) -or ($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
$CurentLoopProgress = @{
ID = $JobID
Activity = "Loading Telemetry for " + $_.FullName
PercentComplete = 75
}
Write-Progress @CurentLoopProgress Write-Progress @CurentLoopProgress
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
$folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting
$folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\")
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
$folder_FileCount = ($FolderPath_Data | measure).Count
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
if ($sql_result.project -ne $null) { if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) {
# found a record, but needs updating - delete old record from DB
$Query = "Delete from FolderData WHERE '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`""
Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
}
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
$folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting
$folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\")
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
$folder_FileCount = ($FolderPath_Data | measure).Count
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) {
#Let's ignore Null Values #Let's ignore Null Values
} }
else { else {
$out = [PSCustomObject]@{ $out = [PSCustomObject]@{
Server = $Server Server = $Server
Project = $ProjectNumber.ToString() Parent = $folder_Parent
Parent = $folder_Parent Path = $Path
Path = $Path FileCount = $folder_FileCount
FileCount = $folder_FileCount FileSize = $folder_FileSize
FileSize = $folder_FileSize FileLastWrite = $folder_LastWrite
FileLastWrite = $folder_LastWrite FileLastAccess = $folder_LastAccess
FileLastAccess = $folder_LastAccess Status = $null # added to support Migration Status
Status = $null # added to support Migration Status fld_MigrationID = $null
fld_MigrationID = $null fld_LastRefresh = (Get-Date).ToShortDateString()
fld_LastRefresh = (Get-Date).ToShortDateString() } | Out-DataTable
} | Out-DataTable
Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
}
} }
$DBConnect.Close()
} }
$DBConnect.Close()
Write-Progress @CurentLoopProgress -Completed Write-Progress @CurentLoopProgress -Completed
} }
Write-Progress @ServerProgress -Completed Write-Progress @ServerProgress -Completed
@ -185,7 +149,7 @@ server,path,path2,etc,status,OriginalServer=RD,OriginalProject=ProjectNumber
#> #>
$DBConnect = New-SqliteConnection -DataSource $PathToDB
<# Get Migration Status <# Get Migration Status
CREATE TABLE "MigrationStatus" ( CREATE TABLE "MigrationStatus" (
"MigrationID" TEXT NOT NULL UNIQUE, "MigrationID" TEXT NOT NULL UNIQUE,
@ -239,61 +203,61 @@ Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
## Kick off GenTables ## Kick off GenTables
$SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql' # $SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run # Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql' # $SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run # Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$DBConnect = New-SqliteConnection -DataSource $PathToDB
foreach ($office in $OfficeList) {
$OfficeServer = $office.Name
$Query = "select * from OfficeDuplicates where Server = '$OfficeServer';" # foreach ($office in $OfficeList) {
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query # $OfficeServer = $office.Name
if ($sql_result) { # $Query = "select * from OfficeDuplicates where Server = '$OfficeServer';"
$OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx" # $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter
} # if ($sql_result) {
} # $OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
# Remove-Item -Path $OutFile -Force
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
# }
# }
# Generate MPE All Xlsx # Generate MPE All Xlsx
$Query = "select * from OfficeDuplicates;" # $Query = "select * from OfficeDuplicates;"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query # $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) { # if ($sql_result) {
$OutFile = $OutputFolder + "\MPE All Duplicates.xlsx" # $OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
Remove-Item -Path $OutFile -Force # Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter # $sql_result | Export-Excel -Path $OutFile -AutoFilter
} # }
# Generate Parent Non-Duplicates # Generate Parent Non-Duplicates
$Query = "select * from NonDupParent;" # $Query = "select * from NonDupParent;"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query # $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) { # if ($sql_result) {
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx" # $OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx"
Remove-Item -Path $OutFile -Force # Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter # $sql_result | Export-Excel -Path $OutFile -AutoFilter
} # }
# Generate Project Non-Duplicates # # Generate Project Non-Duplicates
$Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);" # $Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query # $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) { # if ($sql_result) {
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx" # $OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx"
Remove-Item -Path $OutFile -Force # Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter # $sql_result | Export-Excel -Path $OutFile -AutoFilter
} # }
## Generate Egnyte Data Inventory Sheet ## Generate Egnyte Data Inventory Sheet
@ -307,13 +271,39 @@ catch {
} }
$SQLFile_to_Run = $PSScriptRoot + '\Generate Egnyte Export.sql' $SQLFile_to_Run = $PSScriptRoot + '\Generate Egnyte Export.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Source from DataInventory" $EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Status from DataInventory"
if ($EgnyteExport) { if ($EgnyteExport) {
$EgnyteExport | Export-Excel -Path $OutFile -AutoFilter $EgnyteExport | Export-Excel -Path $OutFile -AutoFilter
} }
$OutFile = $OutputFolder + "\FULL MPE Data Inventory " + (Get-Date -Format FileDate) + ".xlsx"
try {
Remove-Item -Path $OutFile -Force -ErrorAction SilentlyContinue
}
catch {
<#Do this if a terminating exception happens#>
}
$Query = 'Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status, fld_MigrationID,MigrationName from FolderData
LEFT JOIN MigrationStatus on FolderData.fld_MigrationID = MigrationID
-- WHERE FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
ORDER by UNCPath
);'
$FullEgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($FullEgnyteExport) {
$FullEgnyteExport | Export-Excel -Path $OutFile -AutoFilter
}
$DBConnect.Close() $DBConnect.Close()