diff --git a/ArchivedCode/Find-DuplicatePaths.ps1 b/ArchivedCode/Find-DuplicatePaths.ps1 new file mode 100644 index 0000000..410d5cd --- /dev/null +++ b/ArchivedCode/Find-DuplicatePaths.ps1 @@ -0,0 +1,246 @@ +<# +for each server / Office + sanitize project number + Compare against each other office + if matched + get-file counts for each server + report duplicate site / count +#> + + +<# +Migrating to use a SQLLite DB. + +#> + + +$PathToDB = "C:\Users\eeckert\Downloads\EgnyteTesting\Dedupe.SQLite" +$PathToOutput = "C:\Users\eeckert\Downloads\EgnyteTesting\Output" + + + + + +$CSV_ImportFolder = "M:\IT\Egnyte\DuplicateFiles\WorkingRun" +$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv' + +$Office_to_compare_data = $null +foreach ($CSV in $CSV_List) { + $csv_temp = Import-Csv $CSV.FullName + $Office_to_compare_data += $csv_temp + # all Offices loaded into data strcuture 1 +} + +$Office_to_compare_data = $Office_to_compare_data | sort Project, OfficeServer + +$temp_CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Exclude "Output" | sort name -Descending # get list of CSV's to compare to. +$temp_csv_data = $null +foreach ($temp_csv in $temp_CSV_List) { + + $temp_csv_temp = Import-Csv -Path $temp_csv.FullName # load comparison data + $temp_csv_data += $temp_csv_temp | sort Project, OfficeServer + # all offices loaded into comparison data structure +} + + + + +$ServerProgress = @{ + ID = 1 + Activity = "Processing: " + $CSV.OfficeServer + # PercentComplete = 100 + # CurrentOperation = $_.Project +} +Write-Progress @ServerProgress + +$CurrentDate = Get-Date +if ($CurrentDate.DayOfWeek -in ("Monday", "Tuesday", "Wednesday", "Thursday", "Friday") -and $CurrentDate.Hour -GT 17 ) { + $ThreadLimit = 30 +} +elseif ($CurrentDate.DayOfWeek -in ("Saturday", "Sunday")) { + $ThreadLimit = 30 +} +else { + $ThreadLimit = 15 +} + +# foreach ($row in $Office_to_compare_data) { +# $Office_to_compare_data | Foreach-Object { +$Office_to_compare_data | Foreach-Object -ThrottleLimit $ThreadLimit -Parallel { + #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname + + $PSStyle.Progress.View = "Minimal" + $DuplicateObj = $null + $NonDuplicateObj = $null + + $JobID = Get-Random + + $ComparisonProgress = @{ + ID = $JobID + Activity = "Starting: " + $_.OfficeServer + ':' + $_.Project + PercentComplete = 0 + # CurrentOperation = $_.Project + } + Write-Progress @ComparisonProgress + + # if ($_.Project -in $USING:temp_csv_data.Project) { + if ($_.Project -in $USING:temp_csv_data.Project -and $USING:temp_csv_data.OfficeServer -ne $_.OfficeServer) { + $ignoreOffice = $_.OfficeServer + + $duplicates = '' + $duplicates = $USING:temp_csv_data | where -property Project -eq $_.Project | where -Property OfficeServer -ne $_.OfficeServer + # $duplicates = $USING:temp_csv_data | where ((Project -eq $_.Project) -and (OfficeServer -ne $_.OfficeServer)) + + if (($_ -eq $duplicates) -or !($duplicates)) { + + } + else { + + $ComparisonProgress = @{ + ID = $JobID + Activity = "Get Original Data: " + $_.OfficeServer + ':' + $_.Project + PercentComplete = 15 + CurrentOperation = "Getting Original data from " + $_.OfficeServer + } + Write-Progress @ComparisonProgress + + Write-Debug ("Getting file counts from current office...this will take a second...") -Debug + # TODO - identify NON-Duplicated files in these file structures... + # $row_file_count = [System.IO.Directory]::GetFiles($_.FullPath, "*.*", "AllDirectories") | measure + $row_file = get-childitem -path $_.FullPath -recurse -depth 50 -File + $row_file_count = $row_file | measure + $row_file_lastwrite = $row_file | measure LastWriteTime -Maximum + $row_file_lastaccess = $row_file | measure LastAccessTime -Maximum + + foreach ($duplicate in $duplicates) { + + + $ComparisonProgress = @{ + ID = $JobID + Activity = "Get Duplicated data: " + $duplicate.OfficeServer + ':' + $_.Project + PercentComplete = 33 + CurrentOperation = "Now getting duplicate path data from " + $duplicate.OfficeServer + } + Write-Progress @ComparisonProgress + + + $duplicate_file = get-childitem -path $duplicate.FullPath -recurse -depth 50 -File + $duplicate_file_count = $duplicate_file | measure + $duplicate_file_lastwrite = $duplicate_file | measure LastWriteTime -Maximum + $duplicate_file_lastaccess = $duplicate_file | measure LastAccessTime -Maximum + + $ComparisonProgress = @{ + ID = $JobID + Activity = "Processing: " + $duplicate.OfficeServer + ':' + $_.Project + PercentComplete = 66 + # CurrentOperation = $_.Project + } + Write-Progress @ComparisonProgress + + $result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + $ParentPath = $result3.value + + $OriginalProject = "'" + [String]$_.Project + $DuplicateProject = "'" + [String]$duplicate.Project + + if ($row_file_count.Count -ge 1 -and $duplicate_file_count.Count -ge 1) { + $DuplicateObj = [PSCustomObject]@{ + OriginalServer = $_.OfficeServer + OriginalProject = "'" + $_.project + OriginalParent = $ParentPath + OriginalPath = $_.FullPath + OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0 + OriginalFileLastWrite = $row_file_Lastwrite.maximum + OriginalFileAccess = $row_file_lastaccess.maximum + DuplicateServer = $duplicate.OfficeServer + DuplicatePath = $duplicate.FullPath + DuplicateFileCount = $duplicate_file_count.Count ? $duplicate_file_count.count : 0 + DuplicateFileLastWrite = $duplicate_file_Lastwrite.maximum + DuplicateFileAccess = $duplicate_file_lastaccess.maximum + } + + } + elseif ($row_file_count.Count -eq 0 -and $duplicate_file_count.Count -gt 0) { + # Write-Host "File count is 0 on current side, remote server is source" -ForegroundColor DarkBlue + $NonDuplicateObj = [PSCustomObject]@{ + OriginalServer = $duplicate.OfficeServer + OriginalProject = "'" + $duplicate.Project + OriginalPath = $duplicate.FullPath + OriginalFileCount = $duplicate_file_count.Count ? $duplicate_file_count.Count : 0 + OriginalFileLastWrite = $duplicate_file_Lastwrite.maximum + OriginalFileAccess = $duplicate_file_lastaccess.maximum + } + } + elseif ($row_file_count.count -gt 0 -and $duplicate_file_count.Count -eq 0) { + # Write-Host "File count is 0 on one side, not a duplicate..." -ForegroundColor Blue + $NonDuplicateObj = [PSCustomObject]@{ + OriginalServer = $_.OfficeServer + OriginalProject = "'" + $_.Project + OriginalPath = $_.FullPath + OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0 + OriginalFileLastWrite = $row_file_Lastwrite.maximum + OriginalFileAccess = $row_file_lastaccess.maximum + } + + } + + elseif ($_.Project -inotin $Office_to_compare_data.Project) { + $NonDuplicateObj = [PSCustomObject]@{ + OriginalServer = $_.OfficeServer + OriginalProject = "'" + $_.Project + OriginalPath = $_.FullPath + OriginalFileCount = $row_file_count.Count ? $row_file_count.Count : 0 + OriginalFileLastWrite = $row_file_Lastwrite.maximum + OriginalFileAccess = $row_file_lastaccess.maximum + } + + } + + $ComparisonProgress = @{ + ID = $JobID + Activity = "Writing: " + $_.OfficeServer + ':' + $_.Project + PercentComplete = 100 + # CurrentOperation = $_.Project + } + Write-Progress @ComparisonProgress -Completed + + + # Write-Host -ForegroundColor DarkGreen $_.OfficeServer "done, dumping output..." + } + do { + try { + + # $DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result -ErrorAction SilentlyContinue + $DuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $_.OfficeServer + ' Duplicates.csv') -Append -ErrorVariable Dup_CSV_result -ErrorAction SilentlyContinue + } + catch { + <#Do this if a terminating exception happens#> + } + + } while ($Dup_CSV_result.ErrorRecord -ne $null) + } + } + + + # do { + # try { + # $NonDuplicateObj | Export-Csv -Path ($USING:CSV_ImportFolder + '\Output\' + $USING:CSV.Name + 'NonDuplicates.csv') -Append -ErrorVariable Non_Dup_CSV_result -ErrorAction SilentlyContinue + + # } + # catch { + # <#Do this if a terminating exception happens#> + # } + + # } while ($Non_Dup_CSV_Result.ErrorRecord -ne $null) + $ComparisonProgress = @{ + ID = $JobID + Activity = "Done: " + $_.OfficeServer + ':' + $_.Project + PercentComplete = 100 + # CurrentOperation = $_.Project + } + Write-Progress @ComparisonProgress -Completed +} + +# $OutPAth = $CSV_ImportFolder + '\Output\' + $CSV.Name + ' Duplicates.csv' + +Write-Progress @ServerProgress -Completed diff --git a/ArchivedCode/FindNonDupFiles.ps1 b/ArchivedCode/FindNonDupFiles.ps1 new file mode 100644 index 0000000..b619cc2 --- /dev/null +++ b/ArchivedCode/FindNonDupFiles.ps1 @@ -0,0 +1,149 @@ + + +$CSV_ImportFile = 'M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output' +$CSV_ClientList = "M:\IT\Egnyte\DuplicateFiles\Client List.csv" + +$date = Get-Date -Format FileDate +$exportPath_main = "M:\IT\Egnyte\DuplicateFiles\$date NonDuplicates.csv" + +$AllDuplicates_files = Get-ChildItem -Path $CSV_ImportFile -Filter "* Duplicates.csv" -File +$AllDuplicates = @() +foreach ($file in $AllDuplicates_files) { + $temp_val = Import-Csv $file.FullName + $AllDuplicates += $temp_val +} + +$ClientList = Import-csv $CSV_ClientList + +$BaseDrive = 'X:\' +$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}' +$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0 + +$OfficeList | ForEach-Object -ThrottleLimit 3 -Parallel { + #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname + + $XDrivePath = $_.FullName + Write-Host "Processing " $_.FullName -ForegroundColor Green + $ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex + + $All_DupParent_Main = $USING:AllDuplicates.OriginalParent | sort -Unique + $All_DupProject_Main = $USING:AllDuplicates.OriginalProject | sort -Unique + $exportPath = $USING:exportPath_main + + # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. + # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. + + $ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel { #uncomment for threaded mode + # $ProjectFolders | ForEach-Object { #comment for threaded mode + Write-Host "Processing " $_.FullName -ForegroundColor Cyan + $projectNumber = '' + # thank you ChatGPT + $string = $_.FullName + $regex = "\\(\d+(-\d+)*)" + $matches = [regex]::Matches($string, $regex) + $result = "" + foreach ($match in $matches) { + $result += $match.Groups[1].Value + if ($match.Groups[2].Success) { + $result += $match.Groups[2].Value + } + # if ($match.Groups[3].Success) { + # $result += $match.Groups[3].Value + # } + } + + $projectnumber = ("'" + $result) + $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + $ParentPath = $result3.value + + # / Thank you Chat GPT + if ($projectnumber.Length -lt 6 ) { + # write-host $_.FullName " is not a project" -ForegroundColor Cyan + } + elseif ($projectnumber.Length -ge 6) { + $out = $null + $OfficeRegex = "(X:\\[A-Z]+)" + $str = $_.FullName -replace ($OfficeRegex, "") + $OfficeServer = $_.FullName.Substring(1, 4) -replace (':|\\', '') + + $All_DupParent_copy = $USING:All_DupParent_main + + + if ($ParentPath -notin $All_DupParent_copy) { + $str = $ParentPath -replace ($OfficeRegex, "") + $EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/") + + $fileList = Get-ChildItem $_.FullName -Recurse -Depth 1 + $FileSize = $fileList | measure -Sum Length + $totalFiles = $fileList | measure + $LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum + $LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum + + + + $out = [PSCustomObject]@{ + OfficeServer = $OfficeServer + ProjectNumber = $projectNumber + ParentPath = $ParentPath + UNCPath = $ParentPath -replace ('X:', "\\mpe.ca\datadrive") + EgnytePath1 = $EgnytePath1 + # EgnytePath2 = $EgnytePath2 + FileCount = $totalFiles.Count + FolderSize = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB + LastWrite = $LastWriteDate.Maximum + LastAccess = $LastAccessDate.Maximum + Parent = "Yes" + } + do { + try { + + $out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue + } + catch { + <#Do this if a terminating exception happens#> + } + + } while ($CSV_result.ErrorRecord -ne $null) + } + elseif ($projectNumber -notin $USING:All_DupProject_Main) { + $str = $_.FullName -replace ($OfficeRegex, "") + $EgnytePath1 = ("/Shared/N-Data" + $str) -replace ("\\", "/") + + $fileList = Get-ChildItem $_.FullName -Recurse -Depth 50 + $FileSize = $fileList | measure -Sum Length + $totalFiles = $fileList | measure + $LastWriteDate = $fileList | Measure-Object LastWriteTime -Maximum + $LastAccessDate = $fileList | Measure-Object LastAccessTime -Maximum + + if (($FileSize.Sum -gt 0) -or ($totalFiles.Count -gt 0)) { + # no need to do anything if either value is Zero. + $out = [PSCustomObject]@{ + OfficeServer = $OfficeServer + ProjectNumber = $projectNumber + ParentPath = $ParentPath + UNCPath = $_.FullName -replace ('X:', "\\mpe.ca\datadrive") + EgnytePath1 = $EgnytePath1 + # EgnytePath2 = $EgnytePath2 + FileCount = $totalFiles.Count + FolderSize = ($FileSize.Sum / 1024 / 1024 / 1024) # Get's size in GB + LastWrite = $LastWriteDate.Maximum + LastAccess = $LastAccessDate.Maximum + Parent = "No" + } + } + do { + try { + + $out | Export-Csv -Path $USING:exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue + # $out | Export-Csv -Path $exportPath -Append -ErrorVariable CSV_result -ErrorAction SilentlyContinue + } + catch { + <#Do this if a terminating exception happens#> + } + + } while ($CSV_result.ErrorRecord -ne $null) + } + + } + } +} \ No newline at end of file diff --git a/ArchivedCode/ParentSquish.ps1 b/ArchivedCode/ParentSquish.ps1 new file mode 100644 index 0000000..1597824 --- /dev/null +++ b/ArchivedCode/ParentSquish.ps1 @@ -0,0 +1,38 @@ +<# +Import all NonDuplicates. +Find "root" path for each client folder. +if a ROOT folder appears in the Duplicate list (MPE All.CSV), then we can't migrate. +If a root does not appear, we can output that to...something...and these can be added to the migration list without subfolders. + +#> + + +$date = Get-Date -Format FileDate +$ImportPath = "M:\IT\Egnyte\DuplicateFiles\20230906 NonDuplicates.csv" +$DuplicateImportPath = "M:\IT\Egnyte\DuplicateFiles\MPE All.csv" + +$Non_Duplicate_Folders = Import-Csv -Path $ImportPath +$Duplicate_Folders = Import-Csv -Path $DuplicateImportPath + +$dupfolders = @() +foreach ($Dup_Folder in $Duplicate_Folders) { + $Duplicateresult = [Regex]::Matches($Dup_Folder.OriginalPath.Replace('X:\', '\\mpe.ca\datadrive\'), "^(.*[\\\/])") + $dupfolders += $Duplicateresult.Value + + +} +$dupfolders = $dupfolders | sort -Unique +$folder = [PSCustomObject]@() +$Non_Duplicate_Folders | foreach { + + + $result3 = [Regex]::Matches($_.UNCPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + $ParentPath = $result3.value + $out = @() + if ($ParentPath -notin $dupfolders) { + $out += [PSCustomObject]@{ + ParentPath = $ParentPath + } + } + $folder += $out +} diff --git a/DataBaseLoad.ps1 b/DataBaseLoad.ps1 new file mode 100644 index 0000000..a1231ba --- /dev/null +++ b/DataBaseLoad.ps1 @@ -0,0 +1,127 @@ +$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite" +$PathToOutput = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output" +$Refresh = $false + + +$DBExist = Test-Path $PathToDB + +if (!($DBExist)) { + # Create SQLite DB + $Query = "Create Table FolderData (Server TEXT, Project TEXT, Parent TEXT, Path TEXT,FileCount INTEGER,FileSize REAL, FileLastWrite DATETIME, FileLastAccess DATETIME);PRAGMA journal_mode=WAL;" + $QResult = Invoke-SqliteQuery -Database $PathToDB -Query $Query + + +} + +$CSV_ImportFolder = "C:\Users\eeckert\Downloads\EgnyteTesting" +$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv' + +$Office_to_compare_data = $null +foreach ($CSV in $CSV_List) { + $csv_temp = Import-Csv $CSV.FullName + $Office_to_compare_data += $csv_temp + # all Offices loaded into data strcuture 1 +} + +# Load Data into SQL DB +$Office_to_compare_data = $Office_to_compare_data | sort -Property Project, OfficeServer +$Office_to_compare_data | ForEach-Object -ThrottleLimit 15 -parallel { #-ThrottleLimit 5 -parallel + + $ErrorLog = $USING:PathToOutput + '\SQL Error Log.txt' + $DBCopy = $USING:PathToDB #copy DB Path into local runspace + $Refresh_Copy = $USING:Refresh + # $ErrorLog = $PathToOutput + '\SQL Error Log.txt' + # $DBCopy = $PathToDB #copy DB Path into local runspace + # $Refresh_Copy = $Refresh + + $JobID = Get-Random + $ProgressData = @{ + ID = $JobID + Activity = "Analysing:" + $_.OfficeServer + ": " + $_.Project + PercentComplete = 10 + + } + Write-Progress @ProgressData + + $OfficeServer = $_.OfficeServer + $Project = $_.Project + $FullPath = $_.FullPath + + $DBConn = New-SQLiteConnection -Database $DBCopy + + $Query = "SELECT Project FROM FolderData where Project = '$Project';" + $sql_result = Invoke-SqliteQuery -DataSource $DBCopy -Query $Query + + if ($sql_result.Project -eq $null -or $Refresh_Copy -eq $true) { + # No need to re-do what we've done - move on to the next one + + if ($DBConn.State -eq "Open") { + + $result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + + $ProgressData = @{ + ID = $JobID + Activity = "Getting file data for:" + $_.OfficeServer + ": " + $_.Project + PercentComplete = 25 + + } + Write-Progress @ProgressData + + $FolderPath_Data = Get-ChildItem -Path $_.FullPath -File -Depth 50 -Recurse + $folder_Parent = $result3.value + $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024 + $folder_FileCount = ($FolderPath_Data | measure).Count + $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum + $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum + + $ProgressData = @{ + ID = $JobID + Activity = "Writing to DB:" + $_.OfficeServer + ": " + $_.Project + PercentComplete = 75 + + } + Write-Progress @ProgressData + + + ## Build Query and hopefully write to DB + + if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) { + #Let's ignore Null Values + } + else { + + try { + $Query = "Insert into FolderData (Server,Project,Parent,Path,FileCount,FileSize,FileLastWrite,FileLastAccess) VALUES ('$OfficeServer','$Project',`"$folder_Parent`",`"$FullPath`",$folder_FileCount,$folder_FileSize,'$folder_LastWrite','$Folder_LastAccess')" + Invoke-SqliteQuery -Database $DBCopy -Query $Query #-ErrorVariable SQLError + } + catch { + $timestamp = get-date + $errorMessage = $SQLError.ErrorRecord + + $OutString = "[$timestamp] `t $errorMessage `t Query: `t $Query" + + try { + $OutString | Add-Content -Path $ErrorLog + } + catch { + New-Item -Path $ErrorLog -Value $OutString + } + } + } + + } + } + + + + + $DBConn.Close() + $ProgressData = @{ + ID = $JobID + Activity = "Analysing:" + $_.OfficeServer + ": " + $_.Project + PercentComplete = 100 + } + Write-Progress @ProgressData -Completed + +} + diff --git a/Get-ProjectFolders.ps1 b/Get-ProjectFolders.ps1 new file mode 100644 index 0000000..75fd080 --- /dev/null +++ b/Get-ProjectFolders.ps1 @@ -0,0 +1,128 @@ +<# +Script should: +1) get list of all Project Folders +2) Determine if there's content in these folders (exlude shortcuts, lnks, single files) +3) Flag folders with content +4) Output results to file for each server. + +Each Output file should be read by the analysis script to determine locations with duplicate data. + +Output File should show: +X:\CG | ProjectNumber | FullPath | filecount + +#> +$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite" +$BaseDrive = 'X:\' +$ProjectRegex = '^X:\\[A-Z]{2}\W\d{2}\\\d{2}' +$OfficeList = Get-ChildItem -Path $BaseDrive -Directory -Depth 0 + +$OfficeList | ForEach-Object -parallel { + $PathToDB_Copy_Main = $USING:PathToDB + $ServerProgress = @{ + ID = 1 + Activity = "Processing: " + $_.FullName + # PercentComplete = 100 + # CurrentOperation = $_.Project + } + Write-Progress @ServerProgress + #Action that will run in Parallel. Reference the current object via $PSItem and bring in outside variables with $USING:varname + + $XDrivePath = $_.FullName + # Write-Host "Processing " $_.FullName -ForegroundColor Green + $ProjectFolders = get-childitem $_.FullName -Directory -Depth 2 | Where-Object FullName -Match $USING:ProjectRegex + + # $ProjectFolders now contains all folders down to the 3rd level of project number - for Example, 1234-567-001. We can now generate a project number for the output file. + # We need to get a file count for each "last" folder - IE, 001 in the example 1234-567-001. + + $ProjectFolders | ForEach-Object -ThrottleLimit 15 -Parallel { + # Write-Host "Processing " $_.FullName -ForegroundColor Cyan + $PathToDB_Copy = $USING:PathToDB_Copy_Main + + $JobID = Get-Random + $CurentLoopProgress = @{ + ID = $JobID + Activity = "Analyzing " + $_.FullName + PercentComplete = 10 + } + + Write-Progress @CurentLoopProgress + + $projectNumber = '' + # thank you ChatGPT + $string = $_.FullName + $regex = "\\(\d+(-\d+)*)" + $matches = [regex]::Matches($string, $regex) + $result = "" + foreach ($match in $matches) { + $result += $match.Groups[1].Value + if ($match.Groups[2].Success) { + $result += $match.Groups[2].Value + } + # if ($match.Groups[3].Success) { + # $result += $match.Groups[3].Value + # } + } + # / Thank you Chat GPT + # $result = $result -replace ('-','') + + + if ($result.Length -lt 5 ) { + # $CurentLoopProgress = @{ + # ID = $JobID + # Activity = $_.FullName + " Not a Project" + # } + + # Write-Progress @CurentLoopProgress + } + elseif ($result.Length -ge 5) { + + $projectnumber = $result + $DBConnect = New-SqliteConnection -DataSource $PathToDB_Copy + $Query = "Select Project FROM FolderData where Project = '$ProjectNumber'" + $sql_result = Invoke-SqliteQuery -DataSource $PathToDB_Copy -Query $Query + + if ($sql_result.project -eq $null) { + $CurentLoopProgress = @{ + ID = $JobID + Activity = "Loading Telemetry for " + $_.FullName + PercentComplete = 75 + } + + Write-Progress @CurentLoopProgress + + + $result3 = [Regex]::Matches($_.FullName, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent" + $FolderPath_Data = Get-ChildItem -Path $_.FullName -File -Depth 50 -Recurse + $folder_Parent = $result3.value + $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024 + $folder_FileCount = ($FolderPath_Data | measure).Count + $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum + $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum + + if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) { + + #Let's ignore Null Values + } + else { + + $out = [PSCustomObject]@{ + Server = $_.FullName.Substring(1, 4) -replace (':|\\', '') + Project = $ProjectNumber + Parent = $folder_Parent + Path = $_.FullName + FileCount = $folder_FileCount + FileSize = $folder_FileSize + FileLastWrite = $folder_LastWrite + FileLastAccess = $folder_LastAccess + } | Out-DataTable + + Invoke-SQLiteBulkCopy -DataSource $PathToDB_Copy -Table "FolderData" -DataTable $out -Force + + } + } + $DBConnect.Close() + } + Write-Progress @CurentLoopProgress -Completed + } + Write-Progress @ServerProgress -Completed +} -ThrottleLimit 5 \ No newline at end of file