IT/FileServerDedupe
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request 'New_strat' (#1) from New_strat into main

Browse Source 
Reviewed-on: #1
This commit is contained in:
Erik Eckert 2023-10-04 16:00:34 -06:00
commit 187cd43550
3 changed files with 146 additions and 142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Generate Egnyte Export-old.sql Normal file
View File

@ -0,0 +1,24 @@
-- Generate Non-Duplicated PARENT FOLDER List
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS
Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
Where Parent in (
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
)
union
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;

26
Generate Egnyte Export.sql
View File

@ -1,24 +1,14 @@
-- Generate Non-Duplicated PARENT FOLDER List
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS
DROP TABLE IF EXISTS DataInventory;
create table DataInventory as
Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
Where Parent in (
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status from FolderData
WHERE FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
)
union
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;
ORDER by UNCPath
);

152
Get-ProjectFolders.ps1
View File

@ -27,7 +27,7 @@ $PAth_To_Migration_Status = "M:\00 - Egnyte PS Data Inventory\Migration_status\M
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
$OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
$BaseDrive = 'X:\'
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
$ProjectRegex = '^X:\\[A-Z]{2}\W[a-zA-Z0-9 ]*\\'
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
if ($Refresh) {
@ -55,7 +55,7 @@ $OfficeList | ForEach-Object -parallel {
$XDrivePath = $_.FullName
# Write-Host "Processing " $_.FullName -ForegroundColor Green
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 1 | Where-Object FullName -Match $USING:ProjectRegex
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
@ -73,46 +73,19 @@ $OfficeList | ForEach-Object -parallel {
# Write-Progress @CurentLoopProgress
$projectNumber = ''
# thank you ChatGPT
$string = $_.FullName
$regex = "\\(\d+(-\d+)*)"
$matches = [regex]::Matches($string, $regex)
$result = ""
foreach ($match in $matches) {
$result += $match.Groups[1].Value
if ($match.Groups[2].Success) {
$result += $match.Groups[2].Value
}
# if ($match.Groups[3].Success) {
# $result += $match.Groups[3].Value
# }
}
# / Thank you Chat GPT
# $result = $result -replace ('-','')
if ($result.Length -lt 5 ) {
# $CurentLoopProgress = @{
# ID = $JobID
# Activity = $_.FullName + " Not a Project"
# }
# Write-Progress @CurentLoopProgress
}
elseif ($result.Length -ge 5) {
$RefreshDays = -15 - (Get-Random -Maximum 5)
$Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
$Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path
$Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
$projectnumber = $result.tostring()
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
$Query = "Select Project,fld_LastRefresh FROM FolderData where Project = '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`";"
$Query = "Select fld_LastRefresh FROM FolderData where Server = '$Server' AND Path = `"$Path`";"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
if (($sql_result.project -eq $null) -or ($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
if (($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
$CurentLoopProgress = @{
ID = $JobID
Activity = "Loading Telemetry for " + $_.FullName
@ -121,14 +94,6 @@ $OfficeList | ForEach-Object -parallel {
Write-Progress @CurentLoopProgress
if ($sql_result.project -ne $null) {
# found a record, but needs updating - delete old record from DB
$Query = "Delete from FolderData WHERE '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`""
Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
}
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
$folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting
@ -146,7 +111,6 @@ $OfficeList | ForEach-Object -parallel {
$out = [PSCustomObject]@{
Server = $Server
Project = $ProjectNumber.ToString()
Parent = $folder_Parent
Path = $Path
FileCount = $folder_FileCount
@ -163,7 +127,7 @@ $OfficeList | ForEach-Object -parallel {
}
}
$DBConnect.Close()
}
Write-Progress @CurentLoopProgress -Completed
}
Write-Progress @ServerProgress -Completed
@ -185,7 +149,7 @@ server,path,path2,etc,status,OriginalServer=RD,OriginalProject=ProjectNumber
#>
$DBConnect = New-SqliteConnection -DataSource $PathToDB
<# Get Migration Status
CREATE TABLE "MigrationStatus" (
"MigrationID" TEXT NOT NULL UNIQUE,
@ -239,61 +203,61 @@ Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
## Kick off GenTables
$SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
# $SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
# $SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$DBConnect = New-SqliteConnection -DataSource $PathToDB
foreach ($office in $OfficeList) {
$OfficeServer = $office.Name
$Query = "select * from OfficeDuplicates where Server = '$OfficeServer';"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
# foreach ($office in $OfficeList) {
# $OfficeServer = $office.Name
if ($sql_result) {
$OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter
# $Query = "select * from OfficeDuplicates where Server = '$OfficeServer';"
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
}
}
# if ($sql_result) {
# $OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
# Remove-Item -Path $OutFile -Force
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
# }
# }
# Generate MPE All Xlsx
$Query = "select * from OfficeDuplicates;"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
# $Query = "select * from OfficeDuplicates;"
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) {
$OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter
# if ($sql_result) {
# $OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
# Remove-Item -Path $OutFile -Force
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
}
# }
# Generate Parent Non-Duplicates
$Query = "select * from NonDupParent;"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
# $Query = "select * from NonDupParent;"
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) {
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx"
Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter
# if ($sql_result) {
# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx"
# Remove-Item -Path $OutFile -Force
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
}
# }
# Generate Project Non-Duplicates
# # Generate Project Non-Duplicates
$Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
# $Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);"
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($sql_result) {
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx"
Remove-Item -Path $OutFile -Force
$sql_result | Export-Excel -Path $OutFile -AutoFilter
# if ($sql_result) {
# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx"
# Remove-Item -Path $OutFile -Force
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
}
# }
## Generate Egnyte Data Inventory Sheet
@ -307,13 +271,39 @@ catch {
}
$SQLFile_to_Run = $PSScriptRoot + '\Generate Egnyte Export.sql'
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Source from DataInventory"
$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Status from DataInventory"
if ($EgnyteExport) {
$EgnyteExport | Export-Excel -Path $OutFile -AutoFilter
}
$OutFile = $OutputFolder + "\FULL MPE Data Inventory " + (Get-Date -Format FileDate) + ".xlsx"
try {
Remove-Item -Path $OutFile -Force -ErrorAction SilentlyContinue
}
catch {
<#Do this if a terminating exception happens#>
}
$Query = 'Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status, fld_MigrationID,MigrationName from FolderData
LEFT JOIN MigrationStatus on FolderData.fld_MigrationID = MigrationID
-- WHERE FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
ORDER by UNCPath
);'
$FullEgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
if ($FullEgnyteExport) {
$FullEgnyteExport | Export-Excel -Path $OutFile -AutoFilter
}
$DBConnect.Close()