diff --git a/Generate Egnyte Export-old.sql b/Generate Egnyte Export-old.sql new file mode 100644 index 0000000..769ca20 --- /dev/null +++ b/Generate Egnyte Export-old.sql @@ -0,0 +1,24 @@ +-- Generate Non-Duplicated PARENT FOLDER List +DROP TABLE DataInventory; +CREATE TABLE DataInventory AS +Select * from ( + SELECT SERVER, + REPLACE( + Parent, + "X:\","\\mpe.ca\datadrive\") AS UNCPath, + REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath, + sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData + Where Parent in ( + SELECT Parent FROM NonDupParent + ) + AND FolderData.fld_MigrationID IS NULL +GROUP by UNCPath +) +union +SELECT SERVER, + REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath, + REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath, + FileCount, FileSize, 'Project' as Source, Status from FolderData +Where FolderData.Project in ( + SELECT Project FROM NonDupProject) +AND FolderData.fld_MigrationID IS NULL; \ No newline at end of file diff --git a/Generate Egnyte Export.sql b/Generate Egnyte Export.sql index 769ca20..6f8b45b 100644 --- a/Generate Egnyte Export.sql +++ b/Generate Egnyte Export.sql @@ -1,24 +1,14 @@ --- Generate Non-Duplicated PARENT FOLDER List -DROP TABLE DataInventory; -CREATE TABLE DataInventory AS +DROP TABLE IF EXISTS DataInventory; + +create table DataInventory as Select * from ( SELECT SERVER, REPLACE( Parent, "X:\","\\mpe.ca\datadrive\") AS UNCPath, - REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath, - sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData - Where Parent in ( - SELECT Parent FROM NonDupParent - ) - AND FolderData.fld_MigrationID IS NULL + REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath, + sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status from FolderData + WHERE FolderData.fld_MigrationID IS NULL GROUP by UNCPath -) -union -SELECT SERVER, - REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath, - REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath, - FileCount, FileSize, 'Project' as Source, Status from FolderData -Where FolderData.Project in ( - SELECT Project FROM NonDupProject) -AND FolderData.fld_MigrationID IS NULL; \ No newline at end of file +ORDER by UNCPath +); diff --git a/Get-ProjectFolders.ps1 b/Get-ProjectFolders.ps1 index ae91961..ee2c8da 100644 --- a/Get-ProjectFolders.ps1 +++ b/Get-ProjectFolders.ps1 @@ -27,7 +27,7 @@ $PAth_To_Migration_Status = "M:\00 - Egnyte PS Data Inventory\Migration_status\M $PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite" $OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output" $BaseDrive = 'X:\' -$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}' +$ProjectRegex = '^X:\\[A-Z]{2}\W[a-zA-Z0-9 ]*\\' $OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0 if ($Refresh) { @@ -55,7 +55,7 @@ $OfficeList | ForEach-Object -parallel { $XDrivePath = $_.FullName # Write-Host "Processing " $_.FullName -ForegroundColor Green - $ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex + $ProjectFolders = get-childitem $_.FullName -Directory -Depth 1 | Where-Object FullName -Match $USING:ProjectRegex # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. @@ -73,97 +73,61 @@ $OfficeList | ForEach-Object -parallel { # Write-Progress @CurentLoopProgress - $projectNumber = '' + # thank you ChatGPT $string = $_.FullName - $regex = "\\(\d+(-\d+)*)" - $matches = [regex]::Matches($string, $regex) - $result = "" - foreach ($match in $matches) { - $result += $match.Groups[1].Value - if ($match.Groups[2].Success) { - $result += $match.Groups[2].Value + $RefreshDays = -15 - (Get-Random -Maximum 5) + $Server = $_.FullName.Substring(1, 4) -replace (':|\\', '') + $Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path + $Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\") + $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy + $Query = "Select fld_LastRefresh FROM FolderData where Server = '$Server' AND Path = `"$Path`";" + $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query + + + if (($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) { + $CurentLoopProgress = @{ + ID = $JobID + Activity = "Loading Telemetry for " + $_.FullName + PercentComplete = 75 } - # if ($match.Groups[3].Success) { - # $result += $match.Groups[3].Value - # } - } - # / Thank you Chat GPT - # $result = $result -replace ('-','') - - - if ($result.Length -lt 5 ) { - # $CurentLoopProgress = @{ - # ID = $JobID - # Activity = $_.FullName + " Not a Project" - # } - - # Write-Progress @CurentLoopProgress - } - elseif ($result.Length -ge 5) { - $RefreshDays = -15 - (Get-Random -Maximum 5) - - $Server = $_.FullName.Substring(1, 4) -replace (':|\\', '') - $Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path - $Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\") - $projectnumber = $result.tostring() - $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy - $Query = "Select Project,fld_LastRefresh FROM FolderData where Project = '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`";" - $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query - - - if (($sql_result.project -eq $null) -or ($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) { - $CurentLoopProgress = @{ - ID = $JobID - Activity = "Loading Telemetry for " + $_.FullName - PercentComplete = 75 - } - Write-Progress @CurentLoopProgress + Write-Progress @CurentLoopProgress + + $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + $FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse + $folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting + $folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\") + $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024 + $folder_FileCount = ($FolderPath_Data | measure).Count + $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum + $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum - if ($sql_result.project -ne $null) { - # found a record, but needs updating - delete old record from DB - $Query = "Delete from FolderData WHERE '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`"" - Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query - - - } - - $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" - $FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse - $folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting - $folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\") - $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024 - $folder_FileCount = ($FolderPath_Data | measure).Count - $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum - $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum - - if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) { + if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) { - #Let's ignore Null Values - } - else { - - $out = [PSCustomObject]@{ - Server = $Server - Project = $ProjectNumber.ToString() - Parent = $folder_Parent - Path = $Path - FileCount = $folder_FileCount - FileSize = $folder_FileSize - FileLastWrite = $folder_LastWrite - FileLastAccess = $folder_LastAccess - Status = $null # added to support Migration Status - fld_MigrationID = $null - fld_LastRefresh = (Get-Date).ToShortDateString() - } | Out-DataTable - - Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force - - } + #Let's ignore Null Values + } + else { + + $out = [PSCustomObject]@{ + Server = $Server + Parent = $folder_Parent + Path = $Path + FileCount = $folder_FileCount + FileSize = $folder_FileSize + FileLastWrite = $folder_LastWrite + FileLastAccess = $folder_LastAccess + Status = $null # added to support Migration Status + fld_MigrationID = $null + fld_LastRefresh = (Get-Date).ToShortDateString() + } | Out-DataTable + + Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force + } - $DBConnect.Close() } + $DBConnect.Close() + Write-Progress @CurentLoopProgress -Completed } Write-Progress @ServerProgress -Completed @@ -185,7 +149,7 @@ server,path,path2,etc,status,OriginalServer=RD,OriginalProject=ProjectNumber #> - +$DBConnect = New-SqliteConnection -DataSource $PathToDB <# Get Migration Status CREATE TABLE "MigrationStatus" ( "MigrationID" TEXT NOT NULL UNIQUE, @@ -239,61 +203,61 @@ Invoke-SqliteQuery -DataSource $PathToDB -Query $Query ## Kick off GenTables -$SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql' -Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run -$SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql' -Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run +# $SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql' +# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run +# $SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql' +# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run -$DBConnect = New-SqliteConnection -DataSource $PathToDB -foreach ($office in $OfficeList) { - $OfficeServer = $office.Name - $Query = "select * from OfficeDuplicates where Server = '$OfficeServer';" - $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query +# foreach ($office in $OfficeList) { +# $OfficeServer = $office.Name - if ($sql_result) { - $OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx" - Remove-Item -Path $OutFile -Force - $sql_result | Export-Excel -Path $OutFile -AutoFilter +# $Query = "select * from OfficeDuplicates where Server = '$OfficeServer';" +# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query - } -} +# if ($sql_result) { +# $OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx" +# Remove-Item -Path $OutFile -Force +# $sql_result | Export-Excel -Path $OutFile -AutoFilter + +# } +# } # Generate MPE All Xlsx -$Query = "select * from OfficeDuplicates;" -$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query +# $Query = "select * from OfficeDuplicates;" +# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query -if ($sql_result) { - $OutFile = $OutputFolder + "\MPE All Duplicates.xlsx" - Remove-Item -Path $OutFile -Force - $sql_result | Export-Excel -Path $OutFile -AutoFilter +# if ($sql_result) { +# $OutFile = $OutputFolder + "\MPE All Duplicates.xlsx" +# Remove-Item -Path $OutFile -Force +# $sql_result | Export-Excel -Path $OutFile -AutoFilter -} +# } # Generate Parent Non-Duplicates -$Query = "select * from NonDupParent;" -$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query +# $Query = "select * from NonDupParent;" +# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query -if ($sql_result) { - $OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx" - Remove-Item -Path $OutFile -Force - $sql_result | Export-Excel -Path $OutFile -AutoFilter +# if ($sql_result) { +# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx" +# Remove-Item -Path $OutFile -Force +# $sql_result | Export-Excel -Path $OutFile -AutoFilter -} +# } -# Generate Project Non-Duplicates +# # Generate Project Non-Duplicates -$Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);" -$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query +# $Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);" +# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query -if ($sql_result) { - $OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx" - Remove-Item -Path $OutFile -Force - $sql_result | Export-Excel -Path $OutFile -AutoFilter +# if ($sql_result) { +# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx" +# Remove-Item -Path $OutFile -Force +# $sql_result | Export-Excel -Path $OutFile -AutoFilter -} +# } ## Generate Egnyte Data Inventory Sheet @@ -307,13 +271,39 @@ catch { } $SQLFile_to_Run = $PSScriptRoot + '\Generate Egnyte Export.sql' Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run -$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Source from DataInventory" +$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Status from DataInventory" if ($EgnyteExport) { $EgnyteExport | Export-Excel -Path $OutFile -AutoFilter } +$OutFile = $OutputFolder + "\FULL MPE Data Inventory " + (Get-Date -Format FileDate) + ".xlsx" +try { + + Remove-Item -Path $OutFile -Force -ErrorAction SilentlyContinue +} +catch { + <#Do this if a terminating exception happens#> +} +$Query = 'Select * from ( + SELECT SERVER, + REPLACE( + Parent, + "X:\","\\mpe.ca\datadrive\") AS UNCPath, + REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath, + sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status, fld_MigrationID,MigrationName from FolderData + LEFT JOIN MigrationStatus on FolderData.fld_MigrationID = MigrationID +-- WHERE FolderData.fld_MigrationID IS NULL +GROUP by UNCPath +ORDER by UNCPath +);' +$FullEgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query +if ($FullEgnyteExport) { + + $FullEgnyteExport | Export-Excel -Path $OutFile -AutoFilter +} + $DBConnect.Close()