FileServerDedupe/ArchivedCode/DataBaseLoad.ps1

$PathToDB = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Dedupe.SQLite"
$PathToOutput = "M:\IT\Egnyte\DuplicateFiles\WorkingRun\Output"
$Refresh = $false


$DBExist = Test-Path $PathToDB

if (!($DBExist)) {
    # Create SQLite DB
    $Query = "Create Table FolderData (Server TEXT,	Project TEXT, Parent TEXT, Path TEXT,FileCount INTEGER,FileSize REAL, FileLastWrite DATETIME, FileLastAccess DATETIME);PRAGMA journal_mode=WAL;"
    $QResult = Invoke-SqliteQuery -Database $PathToDB -Query $Query


}

$CSV_ImportFolder = "C:\Users\eeckert\Downloads\EgnyteTesting"
$CSV_List = Get-ChildItem -Path $CSV_ImportFolder -Filter '*.csv'

$Office_to_compare_data = $null
foreach ($CSV in $CSV_List) {
    $csv_temp = Import-Csv $CSV.FullName
    $Office_to_compare_data += $csv_temp
    # all Offices loaded into data strcuture 1
}

# Load Data into SQL DB
$Office_to_compare_data = $Office_to_compare_data | sort -Property Project, OfficeServer
$Office_to_compare_data | ForEach-Object -ThrottleLimit 15 -parallel { #-ThrottleLimit 5 -parallel

    $ErrorLog = $USING:PathToOutput + '\SQL Error Log.txt' 
    $DBCopy = $USING:PathToDB #copy DB Path into local runspace
    $Refresh_Copy = $USING:Refresh 
    # $ErrorLog = $PathToOutput + '\SQL Error Log.txt' 
    # $DBCopy = $PathToDB #copy DB Path into local runspace
    # $Refresh_Copy = $Refresh 
    
    $JobID = Get-Random
    $ProgressData = @{
        ID              = $JobID 
        Activity        = "Analysing:" + $_.OfficeServer + ": " + $_.Project
        PercentComplete = 10

    }
    Write-Progress @ProgressData

    $OfficeServer = $_.OfficeServer
    $Project = $_.Project
    $FullPath = $_.FullPath

    $DBConn = New-SQLiteConnection -Database $DBCopy

    $Query = "SELECT Project FROM FolderData where Project = '$Project';"
    $sql_result = Invoke-SqliteQuery -DataSource $DBCopy -Query $Query

    if ($sql_result.Project -eq $null -or $Refresh_Copy -eq $true) {
        # No need to re-do what we've done - move on to the next one

        if ($DBConn.State -eq "Open") {
    
            $result3 = [Regex]::Matches($_.FullPath, "^(.*[\\\/])") # This gets everything up the last slash, thus the "parent"
    
            $ProgressData = @{
                ID              = $JobID 
                Activity        = "Getting file data for:" + $_.OfficeServer + ": " + $_.Project
                PercentComplete = 25
        
            }
            Write-Progress @ProgressData
    
            $FolderPath_Data = Get-ChildItem -Path $_.FullPath -File -Depth 50 -Recurse
            $folder_Parent = $result3.value
            $folder_FileSize = ($FolderPath_Data | Measure -sum Length).sum / 1024 / 1024 / 1024
            $folder_FileCount = ($FolderPath_Data | measure).Count
            $folder_LastWrite = ($FolderPath_Data | measure LastWriteTime -Maximum).Maximum
            $folder_LastAccess = ($FolderPath_Data | measure LastAccessTime -Maximum).Maximum
    
            $ProgressData = @{
                ID              = $JobID 
                Activity        = "Writing to DB:" + $_.OfficeServer + ": " + $_.Project
                PercentComplete = 75
        
            }
            Write-Progress @ProgressData
    
    
            ## Build Query and hopefully write to DB

            if ($folder_FileSize -le 0 -or $folder_FileCount -eq $null) {                   
                #Let's ignore Null Values
            }
            else {
    
                try {
                    $Query = "Insert into FolderData (Server,Project,Parent,Path,FileCount,FileSize,FileLastWrite,FileLastAccess) VALUES ('$OfficeServer','$Project',`"$folder_Parent`",`"$FullPath`",$folder_FileCount,$folder_FileSize,'$folder_LastWrite','$Folder_LastAccess')"
                    Invoke-SqliteQuery -Database $DBCopy -Query $Query #-ErrorVariable SQLError
                }
                catch { 
                    $timestamp = get-date
                    $errorMessage = $SQLError.ErrorRecord
    
                    $OutString = "[$timestamp] `t $errorMessage `t Query: `t $Query"
    
                    try {
                        $OutString | Add-Content -Path $ErrorLog
                    } 
                    catch {
                        New-Item -Path $ErrorLog -Value $OutString
                    }  
                }
            }
    
        }
    }
    
    
    
    
    $DBConn.Close()
    $ProgressData = @{
        ID              = $JobID
        Activity        = "Analysing:" + $_.OfficeServer + ": " + $_.Project
        PercentComplete = 100
    }
    Write-Progress @ProgressData -Completed

}