IT/FileServerDedupe
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

New strategy...dropping to Folder Level

Browse Source
This commit is contained in:
Erik Eckert 2023-10-02 16:06:05 -06:00
parent df72daacc6
commit 59bab0c703
3 changed files with 28 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Generate Egnyte Export-old.sql Normal file
View File

@ -0,0 +1,24 @@
-- Generate Non-Duplicated PARENT FOLDER List
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS
Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
Where Parent in (
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
)
union
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;

25
Generate Egnyte Export.sql
View File

@ -1,24 +1 @@
-- Generate Non-Duplicated PARENT FOLDER List DROP TABLE IF EXISTS DataInventory;
DROP TABLE DataInventory;
CREATE TABLE DataInventory AS
Select * from (
SELECT SERVER,
REPLACE(
Parent,
"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(Parent,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
sum(FileCount) as FileCount, Sum(FileSize) as FileSize, 'Parent' as Source, Status from FolderData
Where Parent in (
SELECT Parent FROM NonDupParent
)
AND FolderData.fld_MigrationID IS NULL
GROUP by UNCPath
)
union
SELECT SERVER,
REPLACE(PATH,"X:\","\\mpe.ca\datadrive\") AS UNCPath,
REPLACE(REPLACE(PATH,("\\mpe.ca\datadrive\" || Server),"/Shared/N-Data"),"\","/") AS EgnytePath,
FileCount, FileSize, 'Project' as Source, Status from FolderData
Where FolderData.Project in (
SELECT Project FROM NonDupProject)
AND FolderData.fld_MigrationID IS NULL;

7
Get-ProjectFolders.ps1
View File

@ -55,7 +55,7 @@ $OfficeList | ForEach-Object -parallel {
$XDrivePath = $_.FullName $XDrivePath = $_.FullName
# Write-Host "Processing " $_.FullName -ForegroundColor Green # Write-Host "Processing " $_.FullName -ForegroundColor Green
$ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex $ProjectFolders = get-childitem $_.FullName -Directory -Depth 1 | Where-Object FullName -Match $USING:ProjectRegex
# $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file.
# We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001.
@ -108,7 +108,7 @@ $OfficeList | ForEach-Object -parallel {
$Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\") $Path = $Path.Replace("X:\", "\\mpe.ca\datadrive\")
$projectnumber = $result.tostring() $projectnumber = $result.tostring()
$DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy
$Query = "Select Project,fld_LastRefresh FROM FolderData where Project = '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`";" $Query = "Select Project,fld_LastRefresh FROM FolderData where Server = '$Server' AND Path = `"$Path`";"
$sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
@ -123,7 +123,7 @@ $OfficeList | ForEach-Object -parallel {
if ($sql_result.project -ne $null) { if ($sql_result.project -ne $null) {
# found a record, but needs updating - delete old record from DB # found a record, but needs updating - delete old record from DB
$Query = "Delete from FolderData WHERE '$ProjectNumber' AND Server = '$Server' AND Path = `"$Path`"" $Query = "Delete from FolderData WHERE Server = '$Server' AND Path = `"$Path`""
Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
@ -146,7 +146,6 @@ $OfficeList | ForEach-Object -parallel {
$out = [PSCustomObject]@{ $out = [PSCustomObject]@{
Server = $Server Server = $Server
Project = $ProjectNumber.ToString()
Parent = $folder_Parent Parent = $folder_Parent
Path = $Path Path = $Path
FileCount = $folder_FileCount FileCount = $folder_FileCount