Merge pull request 'New_strat' (#1) from New_strat into main
Reviewed-on: #1
This commit is contained in:
commit
187cd43550
24
Generate Egnyte Export-old.sql
Normal file
24
Generate Egnyte Export-old.sql
Normal file
@ -0,0 +1,24 @@
|
||||
-- Generate Non-Duplicated PARENT FOLDER List
|
||||
DROP TABLE DataInventory;
|
||||
CREATE TABLE DataInventory AS
|
||||
Select * from (
|
||||
SELECT SERVER,
|
||||
REPLACE(
|
||||
Parent,
|
||||
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
|
||||
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
|
||||
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
|
||||
Where Parent in (
|
||||
SELECT Parent FROM NonDupParent
|
||||
)
|
||||
AND FolderData.fld_MigrationID IS NULL
|
||||
GROUP by UNCPath
|
||||
)
|
||||
union
|
||||
SELECT SERVER,
|
||||
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
|
||||
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
|
||||
FileCount, FileSize, 'Project' as Source, Status from FolderData
|
||||
Where FolderData.Project in (
|
||||
SELECT Project FROM NonDupProject)
|
||||
AND FolderData.fld_MigrationID IS NULL;
|
||||
@ -1,24 +1,14 @@
|
||||
-- Generate Non-Duplicated PARENT FOLDER List
|
||||
DROP TABLE DataInventory;
|
||||
CREATE TABLE DataInventory AS
|
||||
DROP TABLE IF EXISTS DataInventory;
|
||||
|
||||
create table DataInventory as
|
||||
Select * from (
|
||||
SELECT SERVER,
|
||||
REPLACE(
|
||||
Parent,
|
||||
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
|
||||
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
|
||||
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
|
||||
Where Parent in (
|
||||
SELECT Parent FROM NonDupParent
|
||||
)
|
||||
AND FolderData.fld_MigrationID IS NULL
|
||||
REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
|
||||
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status from FolderData
|
||||
WHERE FolderData.fld_MigrationID IS NULL
|
||||
GROUP by UNCPath
|
||||
)
|
||||
union
|
||||
SELECT SERVER,
|
||||
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
|
||||
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
|
||||
FileCount, FileSize, 'Project' as Source, Status from FolderData
|
||||
Where FolderData.Project in (
|
||||
SELECT Project FROM NonDupProject)
|
||||
AND FolderData.fld_MigrationID IS NULL;
|
||||
ORDER by UNCPath
|
||||
);
|
||||
|
||||
@ -27,7 +27,7 @@ $PAth_To_Migration_Status = "M:\00 - Egnyte PS Data Inventory\Migration_status\M
|
||||
$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
|
||||
$OutputFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
|
||||
$BaseDrive = 'X:\'
|
||||
$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
|
||||
$ProjectRegex = '^X:\\[A-Z]{2}\W[a-zA-Z0-9 ]*\\'
|
||||
$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
|
||||
|
||||
if ($Refresh) {
|
||||
@ -55,7 +55,7 @@ $OfficeList | ForEach-Object -parallel {
|
||||
|
||||
$XDrivePath = $_.FullName
|
||||
# Write-Host "Processing " $_.FullName -ForegroundColor Green
|
||||
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
|
||||
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 1 | Where-Object FullName -Match $USING:ProjectRegex
|
||||
|
||||
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
|
||||
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
|
||||
@ -73,97 +73,61 @@ $OfficeList | ForEach-Object -parallel {
|
||||
# Write-Progress @CurentLoopProgress
|
||||
|
||||
|
||||
$projectNumber = ''
|
||||
|
||||
# thank you ChatGPT
|
||||
$string = $_.FullName
|
||||
$regex = "\\(\d+(-\d+)*)"
|
||||
$matches = [regex]::Matches($string, $regex)
|
||||
$result = ""
|
||||
foreach ($match in $matches) {
|
||||
$result += $match.Groups[1].Value
|
||||
if ($match.Groups[2].Success) {
|
||||
$result += $match.Groups[2].Value
|
||||
$RefreshDays = -15 - (Get-Random -Maximum 5)
|
||||
$Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
|
||||
$Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path
|
||||
$Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
|
||||
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
|
||||
$Query = "Select fld_LastRefresh FROM FolderData where Server = '$Server' AND Path = `"$Path`";"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
|
||||
|
||||
|
||||
if (($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
|
||||
$CurentLoopProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Loading Telemetry for " + $_.FullName
|
||||
PercentComplete = 75
|
||||
}
|
||||
# if ($match.Groups[3].Success) {
|
||||
# $result += $match.Groups[3].Value
|
||||
# }
|
||||
}
|
||||
# / Thank you Chat GPT
|
||||
# $result = $result -replace ('-','')
|
||||
|
||||
|
||||
if ($result.Length -lt 5 ) {
|
||||
# $CurentLoopProgress = @{
|
||||
# ID = $JobID
|
||||
# Activity = $_.FullName + " Not a Project"
|
||||
# }
|
||||
|
||||
# Write-Progress @CurentLoopProgress
|
||||
}
|
||||
elseif ($result.Length -ge 5) {
|
||||
$RefreshDays = -15 - (Get-Random -Maximum 5)
|
||||
|
||||
$Server = $_.FullName.Substring(1, 4) -replace (':|\\', '')
|
||||
$Path = $_.FullName.Replace("'", "`'") #SQL Formatted Path
|
||||
$Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
|
||||
$projectnumber = $result.tostring()
|
||||
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
|
||||
$Query = "Select Project,fld_LastRefresh FROM FolderData where Project = '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`";"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
|
||||
|
||||
|
||||
if (($sql_result.project -eq $null) -or ($sql_result.fld_LastRefresh -le (Get-Date).AddDays($RefreshDays).ToShortDateString())) {
|
||||
$CurentLoopProgress = @{
|
||||
ID = $JobID
|
||||
Activity = "Loading Telemetry for " + $_.FullName
|
||||
PercentComplete = 75
|
||||
}
|
||||
|
||||
Write-Progress @CurentLoopProgress
|
||||
Write-Progress @CurentLoopProgress
|
||||
|
||||
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
|
||||
$folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting
|
||||
$folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\")
|
||||
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
|
||||
$folder_FileCount = ($FolderPath_Data | measure).Count
|
||||
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
|
||||
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
|
||||
|
||||
if ($sql_result.project -ne $null) {
|
||||
# found a record, but needs updating - delete old record from DB
|
||||
$Query = "Delete from FolderData WHERE '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`""
|
||||
Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
|
||||
|
||||
|
||||
}
|
||||
|
||||
$result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
|
||||
$FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
|
||||
$folder_Parent = $result3.value.replace("'", "`'") #SQL Formatting
|
||||
$folder_Parent = $folder_Parent.Replace("X:\", "\\mpe.ca\datadrive\")
|
||||
$folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
|
||||
$folder_FileCount = ($FolderPath_Data | measure).Count
|
||||
$folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
|
||||
$folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
|
||||
|
||||
if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) {
|
||||
if ($folder_FileSize -le 0 -or $folder_FileCount -eq 0) {
|
||||
|
||||
#Let's ignore Null Values
|
||||
}
|
||||
else {
|
||||
|
||||
$out = [PSCustomObject]@{
|
||||
Server = $Server
|
||||
Project = $ProjectNumber.ToString()
|
||||
Parent = $folder_Parent
|
||||
Path = $Path
|
||||
FileCount = $folder_FileCount
|
||||
FileSize = $folder_FileSize
|
||||
FileLastWrite = $folder_LastWrite
|
||||
FileLastAccess = $folder_LastAccess
|
||||
Status = $null # added to support Migration Status
|
||||
fld_MigrationID = $null
|
||||
fld_LastRefresh = (Get-Date).ToShortDateString()
|
||||
} | Out-DataTable
|
||||
|
||||
Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
|
||||
|
||||
}
|
||||
#Let's ignore Null Values
|
||||
}
|
||||
else {
|
||||
|
||||
$out = [PSCustomObject]@{
|
||||
Server = $Server
|
||||
Parent = $folder_Parent
|
||||
Path = $Path
|
||||
FileCount = $folder_FileCount
|
||||
FileSize = $folder_FileSize
|
||||
FileLastWrite = $folder_LastWrite
|
||||
FileLastAccess = $folder_LastAccess
|
||||
Status = $null # added to support Migration Status
|
||||
fld_MigrationID = $null
|
||||
fld_LastRefresh = (Get-Date).ToShortDateString()
|
||||
} | Out-DataTable
|
||||
|
||||
Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
|
||||
|
||||
}
|
||||
$DBConnect.Close()
|
||||
}
|
||||
$DBConnect.Close()
|
||||
|
||||
Write-Progress @CurentLoopProgress -Completed
|
||||
}
|
||||
Write-Progress @ServerProgress -Completed
|
||||
@ -185,7 +149,7 @@ server,path,path2,etc,status,OriginalServer=RD,OriginalProject=ProjectNumber
|
||||
|
||||
|
||||
#>
|
||||
|
||||
$DBConnect = New-SqliteConnection -DataSource $PathToDB
|
||||
<# Get Migration Status
|
||||
CREATE TABLE "MigrationStatus" (
|
||||
"MigrationID" TEXT NOT NULL UNIQUE,
|
||||
@ -239,61 +203,61 @@ Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
|
||||
## Kick off GenTables
|
||||
|
||||
$SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
|
||||
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
|
||||
$SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
|
||||
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
|
||||
# $SQLFile_to_Run = $PSScriptRoot + '\GenTables.sql'
|
||||
# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
|
||||
# $SQLFile_to_Run = $PSScriptRoot + '\DuplicatePairing.sql'
|
||||
# Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
|
||||
|
||||
$DBConnect = New-SqliteConnection -DataSource $PathToDB
|
||||
|
||||
foreach ($office in $OfficeList) {
|
||||
$OfficeServer = $office.Name
|
||||
|
||||
$Query = "select * from OfficeDuplicates where Server = '$OfficeServer';"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
# foreach ($office in $OfficeList) {
|
||||
# $OfficeServer = $office.Name
|
||||
|
||||
if ($sql_result) {
|
||||
$OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
|
||||
Remove-Item -Path $OutFile -Force
|
||||
$sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
# $Query = "select * from OfficeDuplicates where Server = '$OfficeServer';"
|
||||
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
|
||||
}
|
||||
}
|
||||
# if ($sql_result) {
|
||||
# $OutFile = $OutputFolder + "\$OfficeServer Duplicates.xlsx"
|
||||
# Remove-Item -Path $OutFile -Force
|
||||
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
|
||||
# }
|
||||
# }
|
||||
# Generate MPE All Xlsx
|
||||
$Query = "select * from OfficeDuplicates;"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
# $Query = "select * from OfficeDuplicates;"
|
||||
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
|
||||
if ($sql_result) {
|
||||
$OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
|
||||
Remove-Item -Path $OutFile -Force
|
||||
$sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
# if ($sql_result) {
|
||||
# $OutFile = $OutputFolder + "\MPE All Duplicates.xlsx"
|
||||
# Remove-Item -Path $OutFile -Force
|
||||
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
|
||||
}
|
||||
# }
|
||||
|
||||
# Generate Parent Non-Duplicates
|
||||
|
||||
$Query = "select * from NonDupParent;"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
# $Query = "select * from NonDupParent;"
|
||||
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
|
||||
if ($sql_result) {
|
||||
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx"
|
||||
Remove-Item -Path $OutFile -Force
|
||||
$sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
# if ($sql_result) {
|
||||
# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Parent Folders.xlsx"
|
||||
# Remove-Item -Path $OutFile -Force
|
||||
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
|
||||
}
|
||||
# }
|
||||
|
||||
|
||||
# Generate Project Non-Duplicates
|
||||
# # Generate Project Non-Duplicates
|
||||
|
||||
$Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);"
|
||||
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
# $Query = "select * from NonDupProject where Path NOT IN (select Path from NonDupParent);"
|
||||
# $sql_result = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
|
||||
if ($sql_result) {
|
||||
$OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx"
|
||||
Remove-Item -Path $OutFile -Force
|
||||
$sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
# if ($sql_result) {
|
||||
# $OutFile = $OutputFolder + "\MPE Non-Duplicates - Project Folders.xlsx"
|
||||
# Remove-Item -Path $OutFile -Force
|
||||
# $sql_result | Export-Excel -Path $OutFile -AutoFilter
|
||||
|
||||
}
|
||||
# }
|
||||
|
||||
## Generate Egnyte Data Inventory Sheet
|
||||
|
||||
@ -307,13 +271,39 @@ catch {
|
||||
}
|
||||
$SQLFile_to_Run = $PSScriptRoot + '\Generate Egnyte Export.sql'
|
||||
Invoke-SqliteQuery -DataSource $PathToDB -InputFile $SQLFile_to_Run
|
||||
$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Source from DataInventory"
|
||||
$EgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query "Select Server,UNCPath,EgnytePath,FileCount,FileSize,Status from DataInventory"
|
||||
|
||||
if ($EgnyteExport) {
|
||||
|
||||
$EgnyteExport | Export-Excel -Path $OutFile -AutoFilter
|
||||
}
|
||||
|
||||
$OutFile = $OutputFolder + "\FULL MPE Data Inventory " + (Get-Date -Format FileDate) + ".xlsx"
|
||||
try {
|
||||
|
||||
Remove-Item -Path $OutFile -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
catch {
|
||||
<#Do this if a terminating exception happens#>
|
||||
}
|
||||
$Query = 'Select * from (
|
||||
SELECT SERVER,
|
||||
REPLACE(
|
||||
Parent,
|
||||
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
|
||||
REPLACE(REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),REPLACE("/Shared/N-Data/" || Server || "_/","_/","_")),"\","/"),"_/","_") AS EgnytePath,
|
||||
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, Status, fld_MigrationID,MigrationName from FolderData
|
||||
LEFT JOIN MigrationStatus on FolderData.fld_MigrationID = MigrationID
|
||||
-- WHERE FolderData.fld_MigrationID IS NULL
|
||||
GROUP by UNCPath
|
||||
ORDER by UNCPath
|
||||
);'
|
||||
$FullEgnyteExport = Invoke-SqliteQuery -DataSource $PathToDB -Query $Query
|
||||
if ($FullEgnyteExport) {
|
||||
|
||||
$FullEgnyteExport | Export-Excel -Path $OutFile -AutoFilter
|
||||
}
|
||||
|
||||
|
||||
|
||||
$DBConnect.Close()
|
||||
|
||||
Loading…
Reference in New Issue
Block a user