migrate to new repo

ArchivedCode/Find-DuplicatePaths.ps1

@@ -0,0 +1,246 @@
+<#
+for each server / Office
+    sanitize project number
+    Compare against each other office
+        if matched
+            get-file counts for each server
+            report duplicate site / count
+#>
+
+
+<#
+Migrating to use a SQLLite DB. 
+
+#>
+
+
+$PathToDB = "C:\Users\eeckert\Downloads\EgnyteTesting\Dedupe.SQLite"
+$PathToOutput = "C:\Users\eeckert\Downloads\EgnyteTesting\Output"
+
+
+
+
+
+$CSV_ImportFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun"
+$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv'
+
+$Office_to_compare_data = $null
+foreach ($CSV in $CSV_List) {
+    $csv_temp = Import-Csv $CSV.FullName
+    $Office_to_compare_data += $csv_temp
+    # all Offices loaded into data strcuture 1
+}
+
+$Office_to_compare_data = $Office_to_compare_data | sort Project, OfficeServer
+
+$temp_CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Exclude "Output" | sort name -Descending # get list of CSV's to compare to.
+$temp_csv_data = $null
+foreach ($temp_csv in $temp_CSV_List) {
+    
+    $temp_csv_temp = Import-Csv -Path $temp_csv.FullName # load comparison data
+    $temp_csv_data += $temp_csv_temp | sort Project, OfficeServer
+    # all offices loaded into comparison data structure
+} 
+
+
+
+
+$ServerProgress = @{
+    ID       = 1
+    Activity = "Processing: " + $CSV.OfficeServer
+    # PercentComplete = 100
+    # CurrentOperation = $_.Project
+}
+Write-Progress @ServerProgress
+
+$CurrentDate = Get-Date
+if ($CurrentDate.DayOfWeek -in ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday") -and $CurrentDate.Hour -GT 17 ) {
+    $ThreadLimit = 30
+}
+elseif ($CurrentDate.DayOfWeek -in ("Saturday", "Sunday")) {
+    $ThreadLimit = 30
+}
+else {
+    $ThreadLimit = 15
+}
+
+# foreach ($row in $Office_to_compare_data) {
+# $Office_to_compare_data | Foreach-Object {
+$Office_to_compare_data | Foreach-Object -ThrottleLimit $ThreadLimit -Parallel {
+    #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
+
+    $PSStyle.Progress.View = "Minimal"
+    $DuplicateObj = $null
+    $NonDuplicateObj = $null
+            
+    $JobID = Get-Random
+
+    $ComparisonProgress = @{
+        ID              = $JobID 
+        Activity        = "Starting: " + $_.OfficeServer + ':' + $_.Project
+        PercentComplete = 0
+        # CurrentOperation = $_.Project
+    }
+    Write-Progress @ComparisonProgress
+
+    # if ($_.Project -in $USING:temp_csv_data.Project) {
+    if ($_.Project -in $USING:temp_csv_data.Project -and $USING:temp_csv_data.OfficeServer -ne $_.OfficeServer) {
+        $ignoreOffice = $_.OfficeServer
+
+        $duplicates = ''
+        $duplicates = $USING:temp_csv_data | where -property Project -eq $_.Project | where -Property OfficeServer -ne $_.OfficeServer
+        # $duplicates = $USING:temp_csv_data | where ((Project -eq $_.Project) -and (OfficeServer -ne $_.OfficeServer))
+
+        if (($_ -eq $duplicates) -or !($duplicates)) {
+            
+        }
+        else {
+
+            $ComparisonProgress = @{
+                ID               = $JobID 
+                Activity         = "Get Original Data: " + $_.OfficeServer + ':' + $_.Project
+                PercentComplete  = 15
+                CurrentOperation = "Getting Original data from " + $_.OfficeServer
+            }
+            Write-Progress @ComparisonProgress
+            
+            Write-Debug ("Getting file counts from current office...this will take a second...") -Debug
+            # TODO - identify NON-Duplicated files in these file structures...
+            # $row_file_count = [System.IO.Directory]::GetFiles($_.FullPath, "*.*", "AllDirectories") | measure
+            $row_file = get-childitem -path $_.FullPath -recurse -depth 50 -File
+            $row_file_count = $row_file | measure
+            $row_file_lastwrite = $row_file | measure LastWriteTime -Maximum
+            $row_file_lastaccess = $row_file | measure LastAccessTime -Maximum
+
+            foreach ($duplicate in $duplicates) {
+
+
+                $ComparisonProgress = @{
+                    ID               = $JobID 
+                    Activity         = "Get Duplicated data: " + $duplicate.OfficeServer + ':' + $_.Project
+                    PercentComplete  = 33
+                    CurrentOperation = "Now getting duplicate path data from " + $duplicate.OfficeServer
+                }
+                Write-Progress @ComparisonProgress
+
+
+                $duplicate_file = get-childitem -path $duplicate.FullPath -recurse -depth 50 -File
+                $duplicate_file_count = $duplicate_file | measure
+                $duplicate_file_lastwrite = $duplicate_file | measure LastWriteTime -Maximum
+                $duplicate_file_lastaccess = $duplicate_file | measure LastAccessTime -Maximum
+
+                $ComparisonProgress = @{
+                    ID              = $JobID 
+                    Activity        = "Processing: " + $duplicate.OfficeServer + ':' + $_.Project
+                    PercentComplete = 66
+                    # CurrentOperation = $_.Project
+                }
+                Write-Progress @ComparisonProgress
+            
+                $result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
+                $ParentPath = $result3.value
+
+                $OriginalProject = "'" + [String]$_.Project
+                $DuplicateProject = "'" + [String]$duplicate.Project
+                
+                if ($row_file_count.Count -ge 1 -and $duplicate_file_count.Count -ge 1) {
+                    $DuplicateObj = [PSCustomObject]@{
+                        OriginalServer         = $_.OfficeServer
+                        OriginalProject        = "'" + $_.project
+                        OriginalParent         = $ParentPath
+                        OriginalPath           = $_.FullPath
+                        OriginalFileCount      = $row_file_count.Count ? $row_file_count.Count : 0
+                        OriginalFileLastWrite  = $row_file_Lastwrite.maximum
+                        OriginalFileAccess     = $row_file_lastaccess.maximum
+                        DuplicateServer        = $duplicate.OfficeServer
+                        DuplicatePath          = $duplicate.FullPath
+                        DuplicateFileCount     = $duplicate_file_count.Count ? $duplicate_file_count.count : 0
+                        DuplicateFileLastWrite = $duplicate_file_Lastwrite.maximum
+                        DuplicateFileAccess    = $duplicate_file_lastaccess.maximum
+                    }
+
+                }
+                elseif ($row_file_count.Count -eq 0 -and $duplicate_file_count.Count -gt 0) {
+                    # Write-Host "File count is 0 on current side, remote server is source" -ForegroundColor DarkBlue
+                    $NonDuplicateObj = [PSCustomObject]@{
+                        OriginalServer        = $duplicate.OfficeServer
+                        OriginalProject       = "'" + $duplicate.Project
+                        OriginalPath          = $duplicate.FullPath
+                        OriginalFileCount     = $duplicate_file_count.Count ? $duplicate_file_count.Count : 0
+                        OriginalFileLastWrite = $duplicate_file_Lastwrite.maximum
+                        OriginalFileAccess    = $duplicate_file_lastaccess.maximum
+                    }
+                }
+                elseif ($row_file_count.count -gt 0 -and $duplicate_file_count.Count -eq 0) {
+                    # Write-Host "File count is 0 on one side, not a duplicate..." -ForegroundColor Blue
+                    $NonDuplicateObj = [PSCustomObject]@{
+                        OriginalServer        = $_.OfficeServer
+                        OriginalProject       = "'" + $_.Project
+                        OriginalPath          = $_.FullPath
+                        OriginalFileCount     = $row_file_count.Count ? $row_file_count.Count : 0
+                        OriginalFileLastWrite = $row_file_Lastwrite.maximum
+                        OriginalFileAccess    = $row_file_lastaccess.maximum
+                    }
+                    
+                }
+                
+                elseif ($_.Project -inotin $Office_to_compare_data.Project) {
+                    $NonDuplicateObj = [PSCustomObject]@{
+                        OriginalServer        = $_.OfficeServer
+                        OriginalProject       = "'" + $_.Project
+                        OriginalPath          = $_.FullPath
+                        OriginalFileCount     = $row_file_count.Count ? $row_file_count.Count : 0
+                        OriginalFileLastWrite = $row_file_Lastwrite.maximum
+                        OriginalFileAccess    = $row_file_lastaccess.maximum
+                    }
+                
+                } 
+
+                $ComparisonProgress = @{
+                    ID              = $JobID
+                    Activity        = "Writing: " + $_.OfficeServer + ':' + $_.Project
+                    PercentComplete = 100
+                    # CurrentOperation = $_.Project
+                }
+                Write-Progress @ComparisonProgress -Completed
+
+
+                # Write-Host -ForegroundColor DarkGreen $_.OfficeServer "done, dumping output..."
+            }
+            do {
+                try {
+                        
+                    # $DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result  -ErrorAction SilentlyContinue
+                    $DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $_.OfficeServer + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result  -ErrorAction SilentlyContinue
+                }
+                catch {
+                    <#Do this if a terminating exception happens#>
+                }
+        
+            } while ($Dup_CSV_result.ErrorRecord -ne $null)
+        }
+    }
+
+
+    # do {
+    #     try {
+    #         $NonDuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + 'NonDuplicates.csv') -Append -ErrorVariable Non_Dup_CSV_result  -ErrorAction SilentlyContinue
+                    
+    #     }
+    #     catch {
+    #         <#Do this if a terminating exception happens#>
+    #     }
+    
+    # } while ($Non_Dup_CSV_Result.ErrorRecord -ne $null)
+    $ComparisonProgress = @{
+        ID              = $JobID
+        Activity        = "Done: " + $_.OfficeServer + ':' + $_.Project
+        PercentComplete = 100
+        # CurrentOperation = $_.Project
+    }
+    Write-Progress @ComparisonProgress -Completed
+}
+
+# $OutPAth = $CSV_ImportFolder + '\Output\' + $CSV.Name + ' Duplicates.csv'
+
+Write-Progress @ServerProgress -Completed

ArchivedCode/FindNonDupFiles.ps1

@@ -0,0 +1,149 @@
+
+
+$CSV_ImportFile = 'M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output'
+$CSV_ClientList = "M:\IT\Egnyte\DuplicateFiles\Client List.csv"
+
+$date = Get-Date -Format FileDate
+$exportPath_main = "M:\IT\Egnyte\DuplicateFiles\$date NonDuplicates.csv"
+
+$AllDuplicates_files = Get-ChildItem -Path $CSV_ImportFile -Filter "* Duplicates.csv" -File
+$AllDuplicates = @()
+foreach ($file in $AllDuplicates_files) {
+    $temp_val = Import-Csv $file.FullName
+    $AllDuplicates += $temp_val
+}
+
+$ClientList = Import-csv $CSV_ClientList
+
+$BaseDrive = 'X:\'
+$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
+$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
+
+$OfficeList | ForEach-Object -ThrottleLimit 3 -Parallel {
+    #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
+
+    $XDrivePath = $_.FullName
+    Write-Host "Processing " $_.FullName -ForegroundColor Green
+    $ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
+
+    $All_DupParent_Main = $USING:AllDuplicates.OriginalParent | sort -Unique
+    $All_DupProject_Main = $USING:AllDuplicates.OriginalProject | sort -Unique
+    $exportPath = $USING:exportPath_main
+
+    # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. 
+    # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. 
+
+    $ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel { #uncomment for threaded mode
+        # $ProjectFolders | ForEach-Object { #comment for threaded mode
+        Write-Host "Processing " $_.FullName -ForegroundColor Cyan
+        $projectNumber = ''
+        # thank you ChatGPT
+        $string = $_.FullName
+        $regex = "\\(\d+(-\d+)*)"
+        $matches = [regex]::Matches($string, $regex)
+        $result = ""
+        foreach ($match in $matches) {
+            $result += $match.Groups[1].Value
+            if ($match.Groups[2].Success) {
+                $result += $match.Groups[2].Value
+            }
+            # if ($match.Groups[3].Success) {
+            #     $result += $match.Groups[3].Value
+            # }
+        }
+
+        $projectnumber = ("'" + $result)
+        $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
+        $ParentPath = $result3.value
+
+        # / Thank you Chat GPT
+        if ($projectnumber.Length -lt 6 ) {
+            # write-host $_.FullName " is not a project" -ForegroundColor Cyan
+        }
+        elseif ($projectnumber.Length -ge 6) {
+            $out = $null
+            $OfficeRegex = "(X:\\[A-Z]+)"
+            $str = $_.FullName -replace ($OfficeRegex, "")
+            $OfficeServer = $_.FullName.Substring(1, 4) -replace (':|\\', '')
+           
+            $All_DupParent_copy = $USING:All_DupParent_main
+        
+
+            if ($ParentPath -notin $All_DupParent_copy) {
+                $str = $ParentPath -replace ($OfficeRegex, "")
+                $EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/")
+
+                $fileList = Get-ChildItem $_.FullName -Recurse -Depth 1
+                $FileSize = $fileList | measure -Sum Length
+                $totalFiles = $fileList | measure
+                $LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum
+                $LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum
+
+
+
+                $out = [PSCustomObject]@{               
+                    OfficeServer  = $OfficeServer
+                    ProjectNumber = $projectNumber
+                    ParentPath    = $ParentPath
+                    UNCPath       = $ParentPath -replace ('X:', "\\mpe.ca\datadrive")
+                    EgnytePath1   = $EgnytePath1
+                    # EgnytePath2   = $EgnytePath2
+                    FileCount     = $totalFiles.Count
+                    FolderSize    = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB
+                    LastWrite     = $LastWriteDate.Maximum
+                    LastAccess    = $LastAccessDate.Maximum
+                    Parent        = "Yes"
+                }
+                do {
+                    try {
+                        
+                        $out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
+                    }
+                    catch {
+                        <#Do this if a terminating exception happens#>
+                    }
+        
+                } while ($CSV_result.ErrorRecord -ne $null)
+            }
+            elseif ($projectNumber -notin $USING:All_DupProject_Main) {
+                $str = $_.FullName -replace ($OfficeRegex, "")
+                $EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/")
+
+                $fileList = Get-ChildItem $_.FullName -Recurse -Depth 50
+                $FileSize = $fileList | measure -Sum Length
+                $totalFiles = $fileList | measure
+                $LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum
+                $LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum
+                    
+                if (($FileSize.Sum -gt 0) -or ($totalFiles.Count -gt 0)) {
+                    # no need to do anything if either value is Zero. 
+                    $out = [PSCustomObject]@{               
+                        OfficeServer  = $OfficeServer
+                        ProjectNumber = $projectNumber
+                        ParentPath    = $ParentPath
+                        UNCPath       = $_.FullName -replace ('X:', "\\mpe.ca\datadrive")
+                        EgnytePath1   = $EgnytePath1
+                        # EgnytePath2   = $EgnytePath2
+                        FileCount     = $totalFiles.Count
+                        FolderSize    = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB
+                        LastWrite     = $LastWriteDate.Maximum
+                        LastAccess    = $LastAccessDate.Maximum
+                        Parent        = "No"
+                    }
+                }
+                do {
+                    try {
+                        
+                        $out | Export-Csv -Path $USING:exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
+                        # $out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue
+                    }
+                    catch {
+                        <#Do this if a terminating exception happens#>
+                    }
+        
+                } while ($CSV_result.ErrorRecord -ne $null)
+            }
+              
+        }
+    }
+}

ArchivedCode/ParentSquish.ps1

@@ -0,0 +1,38 @@
+<#
+Import all NonDuplicates.
+Find "root" path for each client folder.
+if a ROOT folder appears in the Duplicate list (MPE All.CSV), then we can't migrate. 
+If a root does not appear, we can output that to...something...and these can be added to the migration list without subfolders. 
+
+#>
+
+
+$date = Get-Date -Format FileDate
+$ImportPath = "M:\IT\Egnyte\DuplicateFiles\20230906 NonDuplicates.csv"
+$DuplicateImportPath = "M:\IT\Egnyte\DuplicateFiles\MPE All.csv"
+
+$Non_Duplicate_Folders = Import-Csv -Path $ImportPath
+$Duplicate_Folders = Import-Csv -Path $DuplicateImportPath
+
+$dupfolders = @()
+foreach ($Dup_Folder in $Duplicate_Folders) {
+    $Duplicateresult = [Regex]::Matches($Dup_Folder.OriginalPath.Replace('X:\', '\\mpe.ca\datadrive\'), "^(.*[\\\/])")
+    $dupfolders += $Duplicateresult.Value
+    
+    
+}
+$dupfolders = $dupfolders | sort -Unique
+$folder = [PSCustomObject]@()
+$Non_Duplicate_Folders | foreach {
+    
+
+    $result3 = [Regex]::Matches($_.UNCPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
+    $ParentPath = $result3.value
+    $out = @()
+    if ($ParentPath -notin $dupfolders) {
+        $out += [PSCustomObject]@{
+            ParentPath = $ParentPath
+        }
+    }
+    $folder += $out
+}

DataBaseLoad.ps1

@@ -0,0 +1,127 @@
+$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
+$PathToOutput = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
+$Refresh = $false
+
+
+$DBExist = Test-Path $PathToDB
+
+if (!($DBExist)) {
+    # Create SQLite DB
+    $Query = "Create Table FolderData (Server TEXT,	Project TEXT, Parent TEXT, Path TEXT,FileCount INTEGER,FileSize REAL, FileLastWrite DATETIME, FileLastAccess DATETIME);PRAGMA journal_mode=WAL;"
+    $QResult = Invoke-SqliteQuery -Database $PathToDB -Query $Query
+
+
+}
+
+$CSV_ImportFolder = "C:\Users\eeckert\Downloads\EgnyteTesting"
+$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv'
+
+$Office_to_compare_data = $null
+foreach ($CSV in $CSV_List) {
+    $csv_temp = Import-Csv $CSV.FullName
+    $Office_to_compare_data += $csv_temp
+    # all Offices loaded into data strcuture 1
+}
+
+# Load Data into SQL DB
+$Office_to_compare_data = $Office_to_compare_data | sort -Property Project, OfficeServer
+$Office_to_compare_data | ForEach-Object -ThrottleLimit 15 -parallel { #-ThrottleLimit 5 -parallel
+
+    $ErrorLog = $USING:PathToOutput + '\SQL Error Log.txt' 
+    $DBCopy = $USING:PathToDB #copy DB Path into local runspace
+    $Refresh_Copy = $USING:Refresh 
+    # $ErrorLog = $PathToOutput + '\SQL Error Log.txt' 
+    # $DBCopy = $PathToDB #copy DB Path into local runspace
+    # $Refresh_Copy = $Refresh 
+    
+    $JobID = Get-Random
+    $ProgressData = @{
+        ID              = $JobID 
+        Activity        = "Analysing:" + $_.OfficeServer + ": " + $_.Project
+        PercentComplete = 10
+
+    }
+    Write-Progress @ProgressData
+
+    $OfficeServer = $_.OfficeServer
+    $Project = $_.Project
+    $FullPath = $_.FullPath
+
+    $DBConn = New-SQLiteConnection -Database $DBCopy
+
+    $Query = "SELECT Project FROM FolderData where Project = '$Project';"
+    $sql_result = Invoke-SqliteQuery -DataSource $DBCopy -Query $Query
+
+    if ($sql_result.Project -eq $null -or $Refresh_Copy -eq $true) {
+        # No need to re-do what we've done - move on to the next one
+
+        if ($DBConn.State -eq "Open") {
+    
+            $result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
+    
+            $ProgressData = @{
+                ID              = $JobID 
+                Activity        = "Getting file data for:" + $_.OfficeServer + ": " + $_.Project
+                PercentComplete = 25
+        
+            }
+            Write-Progress @ProgressData
+    
+            $FolderPath_Data = Get-ChildItem -Path $_.FullPath -File -Depth 50 -Recurse
+            $folder_Parent = $result3.value
+            $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
+            $folder_FileCount = ($FolderPath_Data | measure).Count
+            $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
+            $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
+    
+            $ProgressData = @{
+                ID              = $JobID 
+                Activity        = "Writing to DB:" + $_.OfficeServer + ": " + $_.Project
+                PercentComplete = 75
+        
+            }
+            Write-Progress @ProgressData
+    
+    
+            ## Build Query and hopefully write to DB
+
+            if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) {                   
+                #Let's ignore Null Values
+            }
+            else {
+    
+                try {
+                    $Query = "Insert into FolderData (Server,Project,Parent,Path,FileCount,FileSize,FileLastWrite,FileLastAccess) VALUES ('$OfficeServer','$Project',`"$folder_Parent`",`"$FullPath`",$folder_FileCount,$folder_FileSize,'$folder_LastWrite','$Folder_LastAccess')"
+                    Invoke-SqliteQuery -Database $DBCopy -Query $Query #-ErrorVariable SQLError
+                }
+                catch { 
+                    $timestamp = get-date
+                    $errorMessage = $SQLError.ErrorRecord
+    
+                    $OutString = "[$timestamp] `t $errorMessage `t Query: `t $Query"
+    
+                    try {
+                        $OutString | Add-Content -Path $ErrorLog
+                    } 
+                    catch {
+                        New-Item -Path $ErrorLog -Value $OutString
+                    }  
+                }
+            }
+    
+        }
+    }
+    
+    
+    
+    
+    $DBConn.Close()
+    $ProgressData = @{
+        ID              = $JobID
+        Activity        = "Analysing:" + $_.OfficeServer + ": " + $_.Project
+        PercentComplete = 100
+    }
+    Write-Progress @ProgressData -Completed
+
+}
+

Get-ProjectFolders.ps1

@@ -0,0 +1,128 @@
+<#
+Script should:
+1) get list of all Project Folders 
+2) Determine if there's content in these folders (exlude shortcuts, lnks, single files)
+3) Flag folders with content 
+4) Output results to file for each server.
+
+Each Output file should be read by the analysis script to determine locations with duplicate data. 
+
+Output File should show:
+X:\CG   |   ProjectNumber    |  FullPath | filecount
+
+#>
+$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
+$BaseDrive = 'X:\'
+$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}'
+$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0
+
+$OfficeList | ForEach-Object -parallel {
+    $PathToDB_Copy_Main = $USING:PathToDB
+    $ServerProgress = @{
+        ID       = 1
+        Activity = "Processing: " + $_.FullName
+        # PercentComplete = 100
+        # CurrentOperation = $_.Project
+    }
+    Write-Progress @ServerProgress
+    #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname
+
+    $XDrivePath = $_.FullName
+    # Write-Host "Processing " $_.FullName -ForegroundColor Green
+    $ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex
+
+    # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. 
+    # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. 
+
+    $ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel {
+        # Write-Host "Processing " $_.FullName -ForegroundColor Cyan
+        $PathToDB_Copy = $USING:PathToDB_Copy_Main
+
+        $JobID = Get-Random
+        $CurentLoopProgress = @{
+            ID              = $JobID
+            Activity        = "Analyzing " + $_.FullName
+            PercentComplete = 10
+        }
+
+        Write-Progress @CurentLoopProgress
+
+        $projectNumber = ''
+        # thank you ChatGPT
+        $string = $_.FullName
+        $regex = "\\(\d+(-\d+)*)"
+        $matches = [regex]::Matches($string, $regex)
+        $result = ""
+        foreach ($match in $matches) {
+            $result += $match.Groups[1].Value
+            if ($match.Groups[2].Success) {
+                $result += $match.Groups[2].Value
+            }
+            # if ($match.Groups[3].Success) {
+            #     $result += $match.Groups[3].Value
+            # }
+        }
+        # / Thank you Chat GPT
+        # $result = $result -replace ('-','')
+
+
+        if ($result.Length -lt 5 ) {
+            # $CurentLoopProgress = @{
+            #     ID       = $JobID
+            #     Activity = $_.FullName + " Not a Project"
+            # }
+
+            # Write-Progress @CurentLoopProgress
+        }
+        elseif ($result.Length -ge 5) {
+
+            $projectnumber = $result
+            $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy 
+            $Query = "Select Project FROM FolderData where Project = '$ProjectNumber'"
+            $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query
+
+            if ($sql_result.project -eq $null) {
+                $CurentLoopProgress = @{
+                    ID              = $JobID
+                    Activity        = "Loading Telemetry for " + $_.FullName
+                    PercentComplete = 75
+                }
+        
+                Write-Progress @CurentLoopProgress
+    
+
+                $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
+                $FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse
+                $folder_Parent = $result3.value
+                $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
+                $folder_FileCount = ($FolderPath_Data | measure).Count
+                $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
+                $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
+                
+                if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) {
+                    
+                    #Let's ignore Null Values
+                }
+                else {
+                    
+                    $out = [PSCustomObject]@{
+                        Server         = $_.FullName.Substring(1, 4) -replace (':|\\', '')
+                        Project        = $ProjectNumber
+                        Parent         = $folder_Parent
+                        Path           = $_.FullName
+                        FileCount      = $folder_FileCount
+                        FileSize       = $folder_FileSize
+                        FileLastWrite  = $folder_LastWrite
+                        FileLastAccess = $folder_LastAccess 
+                    } | Out-DataTable
+                    
+                    Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force
+
+                }
+            }
+            $DBConnect.Close()
+        }
+        Write-Progress @CurentLoopProgress -Completed
+    }
+    Write-Progress @ServerProgress -Completed
+} -ThrottleLimit 5