From 9b102f9de2e49d0a00da01b606318e5098562d96 Mon Sep 17 00:00:00 2001 From: felicijus Date: Wed, 15 Oct 2025 18:51:45 +0200 Subject: [PATCH 01/13] refactor(virustotal): move VirusTotal functionality into a lib --- lib/virustotal.ps1 | 311 ++++++++++++++++++++++++++++++++++ libexec/scoop-virustotal.ps1 | 319 +---------------------------------- 2 files changed, 318 insertions(+), 312 deletions(-) create mode 100644 lib/virustotal.ps1 diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 new file mode 100644 index 0000000000..6595e5a77f --- /dev/null +++ b/lib/virustotal.ps1 @@ -0,0 +1,311 @@ +. "$PSScriptRoot\json.ps1" # 'json_path' +. "$PSScriptRoot\download.ps1" # 'hash_for_url' + +# Error codes +$script:_ERR_UNSAFE = 2 +$script:_ERR_EXCEPTION = 4 +$script:_ERR_NO_INFO = 8 +$script:_ERR_NO_API_KEY = 16 + +# Global state variables +$script:requests = 0 +$script:explained_rate_limit_sleeping = $False +$script:exit_code = 0 + +function ConvertTo-VirusTotalUrlId ($url) { + $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) + $url_id = $url_id -replace '\+', '-' + $url_id = $url_id -replace '/', '_' + $url_id = $url_id -replace '=', '' + $url_id +} + +function Get-VirusTotalResultByHash ($hash, $url, $app, $api_key) { + $hash = $hash.ToLower() + $api_url = "https://www.virustotal.com/api/v3/files/$hash" + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + } + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $stats = json_path $result '$.data.attributes.last_analysis_stats' + [int]$malicious = json_path $stats '$.malicious' + [int]$suspicious = json_path $stats '$.suspicious' + [int]$timeout = json_path $stats '$.timeout' + [int]$undetected = json_path $stats '$.undetected' + [int]$unsafe = $malicious + $suspicious + [int]$total = $unsafe + $undetected + [int]$fileSize = json_path $result '$.data.attributes.size' + $report_hash = json_path $result '$.data.attributes.sha256' + $report_url = "https://www.virustotal.com/gui/file/$report_hash" + if ($total -eq 0) { + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'UrlReport.Url' = $null + } + } else { + $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value + switch ($unsafe) { + 0 { success "$app`: $unsafe/$total, see $report_url" } + 1 { warn "$app`: $unsafe/$total, see $report_url" } + 2 { warn "$app`: $unsafe/$total, see $report_url" } + Default { warn "$([char]0x1b)[31m$app`: $unsafe/$total, see $report_url$([char]0x1b)[0m" } + } + $maliciousResults = $vendorResults | + Where-Object -Property category -EQ 'malicious' | + Select-Object -ExpandProperty engine_name + $suspiciousResults = $vendorResults | + Where-Object -Property category -EQ 'suspicious' | + Select-Object -ExpandProperty engine_name + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $hash + 'App.HashType' = $null + 'App.Size' = filesize $fileSize + 'FileReport.Url' = $report_url + 'FileReport.Hash' = $report_hash + 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } + 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } + 'FileReport.Timeout' = $timeout + 'FileReport.Undetected' = $undetected + 'UrlReport.Url' = $null + } + } + if ($unsafe -gt 0) { + $Script:exit_code = $exit_code -bor $script:_ERR_UNSAFE + } +} + +function Get-VirusTotalResultByUrl ($url, $app, $api_key) { + $id = ConvertTo-VirusTotalUrlId $url + $api_url = "https://www.virustotal.com/api/v3/urls/$id" + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + } + $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing + $result = $response.Content + $id = json_path $result '$.data.id' + $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null + $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null + $url_report_url = "https://www.virustotal.com/gui/url/$id" + info "$app`: Url report found." + if (!$hash) { + if (!$last_analysis_date) { + info "$app`: Analysis in progress." + } else { + info "$app`: Related file report not found." + warn "$app`: Manual file upload is required (instead of url submission)." + } + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null + } + } else { + info "$app`: Related file report found." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $hash + } + } +} + +# Submit-ToVirusTotal +# - $url: where file to check can be downloaded +# - $app: Name of the application (used for reporting) +# - $do_scan: [boolean flag] whether to actually submit to VirusTotal +# This is a parameter instead of conditionnally calling +# the function to consolidate the warning message +# - $api_key: VirusTotal API key +# - $retrying: [boolean] Optional, for internal use to retry +# submitting the file after a delay if the rate limit is +# exceeded, without risking an infinite loop (as stack +# overflow) if the submission keeps failing. +function Submit-ToVirusTotal ($url, $app, $do_scan, $api_key, $retrying = $False) { + if (!$do_scan) { + warn "$app`: not found`: you can manually submit $url" + return + } + + try { + $script:requests += 1 + + $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) + $api_url = 'https://www.virustotal.com/api/v3/urls' + $content_type = 'application/x-www-form-urlencoded' + $headers = @{ + 'Accept' = 'application/json' + 'x-apikey' = $api_key + 'Content-Type' = $content_type + } + $body = "url=$encoded_url" + $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing + if ($result.StatusCode -eq 200) { + $id = ((json_path $result '$.data.id') -split '-')[1] + $url_report_url = "https://www.virustotal.com/gui/url/$id" + $fileSize = Get-RemoteFileSize $url + if ($fileSize -gt 80000000) { + info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." + } + info "$app`: Analysis in progress." + [PSCustomObject] @{ + 'App.Name' = $app + 'App.Url' = $url + 'App.Hash' = $null + 'App.HashType' = $null + 'FileReport.Url' = $null + 'UrlReport.Url' = $url_report_url + 'UrlReport.Hash' = $null + } + return + } + + # EAFP: submission failed -> sleep, then retry + if (!$retrying) { + if (!$script:explained_rate_limit_sleeping) { + info 'VirusTotal API has rate limits. Waiting between requests...' + $script:explained_rate_limit_sleeping = $True + } + Start-Sleep -s (60 + $script:requests) + Submit-ToVirusTotal $url $app $do_scan $api_key $True + } else { + warn "$app`: VirusTotal submission of $url failed`:`n" + + "`tAPI returned $($result.StatusCode) after retrying" + } + } catch [Exception] { + warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" + return + } +} + +function Get-VirusTotalApiKey { + $api_key = get_config VIRUSTOTAL_API_KEY + if (!$api_key) { + abort ("VirusTotal API key is not configured`n" + + " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + + " scoop config virustotal_api_key ") $_ERR_NO_API_KEY + } + return $api_key +} + +function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { + [int]$index = 0 + $urls = script:url $manifest $architecture + $urls | ForEach-Object { + $url = $_ + $index++ + if ($urls.GetType().IsArray) { + info "$app`: url $index" + } + $hash = hash_for_url $manifest $url $architecture + + try { + $isHashUnsupported = $false + if ($hash -match '(?[^:]+):(?.*)') { + $algo = $matches.algo + $hash = $matches.hash + if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { + $hash = $null + $isHashUnsupported = $true + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." + } + } elseif ($hash) { + $algo = 'sha256' + } + if ($hash) { + $file_report = Get-VirusTotalResultByHash $hash $url $app $api_key + $file_report.'App.HashType' = $algo + $file_report + return + } elseif (!$isHashUnsupported) { + warn "$app`: Hash not found. Will search by url instead." + } + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true + warn "$app`: File report not found. Will search by url instead." + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } + + try { + $url_report = Get-VirusTotalResultByUrl $url $app $api_key + $url_report.'App.Hash' = $hash + $url_report.'App.HashType' = $matches['algo'] + if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + if ($file_report.'FileReport.Hash' -ieq $matches['hash']) { + $file_report.'App.HashType' = $matches['algo'] + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + return $file_report + } + } catch { + warn "$app`: Unable to get file report for $($url_report.'UrlReport.Hash')" + } + } + if (!$url_report.'UrlReport.Hash') { + Submit-ToVirusTotal $url $app $scan $api_key + return $url_report + } + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + return + } else { + warn "$app`: VirusTotal URL report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } + + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + $file_report.'App.Hash' = $hash + $file_report.'App.HashType' = $matches['algo'] + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + $file_report + warn "$app`: Unable to check hash match for $url" + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + $url_report + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" + } + return + } + } + } +} diff --git a/libexec/scoop-virustotal.ps1 b/libexec/scoop-virustotal.ps1 index d56eb6e8b4..02a427d58b 100644 --- a/libexec/scoop-virustotal.ps1 +++ b/libexec/scoop-virustotal.ps1 @@ -29,14 +29,13 @@ # -p, --passthru Return reports as objects . "$PSScriptRoot\..\lib\getopt.ps1" +. "$PSScriptRoot\..\lib\virustotal.ps1" . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' -. "$PSScriptRoot\..\lib\json.ps1" # 'json_path' -. "$PSScriptRoot\..\lib\download.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\depends.ps1" # 'Get-Dependency' $opt, $apps, $err = getopt $args 'asnup' @('all', 'scan', 'no-depends', 'no-update-scoop', 'passthru') -if ($err) { error "scoop virustotal: $err"; exit 1 } +if ($err) { "scoop virustotal: $err"; exit 1 } $all = $apps -eq '*' -or $opt.a -or $opt.all if (!$apps -and !$all) { my_usage; exit 1 } $architecture = Get-DefaultArchitecture @@ -57,214 +56,8 @@ if (!$opt.n -and !$opt.'no-depends') { $apps = $apps | Get-Dependency -Architecture $architecture | Select-Object -Unique } -$_ERR_UNSAFE = 2 -$_ERR_EXCEPTION = 4 -$_ERR_NO_INFO = 8 -$_ERR_NO_API_KEY = 16 - -$exit_code = 0 - -# Global API key: -$api_key = get_config VIRUSTOTAL_API_KEY -if (!$api_key) { - abort ("VirusTotal API key is not configured`n" + - " You could get one from https://www.virustotal.com/gui/my-apikey and set with`n" + - " scoop config virustotal_api_key ") $_ERR_NO_API_KEY -} - -# Global flag to explain only once about sleep between requests -$explained_rate_limit_sleeping = $False - -# Requests counter to slow down requests submitted to VirusTotal as -# script execution progresses -$requests = 0 - -Function ConvertTo-VirusTotalUrlId ($url) { - $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) - $url_id = $url_id -replace '\+', '-' - $url_id = $url_id -replace '/', '_' - $url_id = $url_id -replace '=', '' - $url_id -} - -Function Get-VirusTotalResultByHash ($hash, $url, $app) { - $hash = $hash.ToLower() - $api_url = "https://www.virustotal.com/api/v3/files/$hash" - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing - $result = $response.Content - $stats = json_path $result '$.data.attributes.last_analysis_stats' - [int]$malicious = json_path $stats '$.malicious' - [int]$suspicious = json_path $stats '$.suspicious' - [int]$timeout = json_path $stats '$.timeout' - [int]$undetected = json_path $stats '$.undetected' - [int]$unsafe = $malicious + $suspicious - [int]$total = $unsafe + $undetected - [int]$fileSize = json_path $result '$.data.attributes.size' - $report_hash = json_path $result '$.data.attributes.sha256' - $report_url = "https://www.virustotal.com/gui/file/$report_hash" - if ($total -eq 0) { - info "$app`: Analysis in progress." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $hash - 'App.HashType' = $null - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $report_url - 'FileReport.Hash' = $report_hash - 'UrlReport.Url' = $null - } - } else { - $vendorResults = (ConvertFrom-Json((json_path $result '$.data.attributes.last_analysis_results'))).PSObject.Properties.Value - switch ($unsafe) { - 0 { - success "$app`: $unsafe/$total, see $report_url" - } - 1 { - warn "$app`: $unsafe/$total, see $report_url" - } - 2 { - warn "$app`: $unsafe/$total, see $report_url" - } - Default { - warn "$([char]0x1b)[31m$app`: $unsafe/$total, see $report_url$([char]0x1b)[0m" - } - } - $maliciousResults = $vendorResults | - Where-Object -Property category -EQ 'malicious' | - Select-Object -ExpandProperty engine_name - $suspiciousResults = $vendorResults | - Where-Object -Property category -EQ 'suspicious' | - Select-Object -ExpandProperty engine_name - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $hash - 'App.HashType' = $null - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $report_url - 'FileReport.Hash' = $report_hash - 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } - 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } - 'FileReport.Timeout' = $timeout - 'FileReport.Undetected' = $undetected - 'UrlReport.Url' = $null - } - } - if ($unsafe -gt 0) { - $Script:exit_code = $exit_code -bor $_ERR_UNSAFE - } -} - -Function Get-VirusTotalResultByUrl ($url, $app) { - $id = ConvertTo-VirusTotalUrlId $url - $api_url = "https://www.virustotal.com/api/v3/urls/$id" - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $response = Invoke-WebRequest -Uri $api_url -Method GET -Headers $headers -UseBasicParsing - $result = $response.Content - $id = json_path $result '$.data.id' - $hash = json_path $result '$.data.attributes.last_http_response_content_sha256' 6>$null - $last_analysis_date = json_path $result '$.data.attributes.last_analysis_date' 6>$null - $url_report_url = "https://www.virustotal.com/gui/url/$id" - info "$app`: Url report found." - if (!$hash) { - if (!$last_analysis_date) { - info "$app`: Analysis in progress." - } else { - info "$app`: Related file report not found." - warn "$app`: Manual file upload is required (instead of url submission)." - } - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $null - 'App.HashType' = $null - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - 'UrlReport.Hash' = $null - } - } else { - info "$app`: Related file report found." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Hash' = $null - 'App.HashType' = $null - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - 'UrlReport.Hash' = $hash - } - } -} - -# Submit-ToVirusTotal -# - $url: where file to check can be downloaded -# - $app: Name of the application (used for reporting) -# - $do_scan: [boolean flag] whether to actually submit to VirusTotal -# This is a parameter instead of conditionnally calling -# the function to consolidate the warning message -# - $retrying: [boolean] Optional, for internal use to retry -# submitting the file after a delay if the rate limit is -# exceeded, without risking an infinite loop (as stack -# overflow) if the submission keeps failing. -Function Submit-ToVirusTotal ($url, $app, $do_scan, $retrying = $False) { - if (!$do_scan) { - warn "$app`: not found`: you can manually submit $url" - return - } - - try { - $requests += 1 - - $encoded_url = [System.Web.HttpUtility]::UrlEncode($url) - $api_url = 'https://www.virustotal.com/api/v3/urls' - $content_type = 'application/x-www-form-urlencoded' - $headers = @{} - $headers.Add('Accept', 'application/json') - $headers.Add('x-apikey', $api_key) - $headers.Add('Content-Type', $content_type) - $body = "url=$encoded_url" - $result = Invoke-WebRequest -Uri $api_url -Method POST -Headers $headers -ContentType $content_type -Body $body -UseBasicParsing - if ($result.StatusCode -eq 200) { - $id = ((json_path $result '$.data.id') -split '-')[1] - $url_report_url = "https://www.virustotal.com/gui/url/$id" - $fileSize = Get-RemoteFileSize $url - if ($fileSize -gt 80000000) { - info "$app`: Remote file size: $(filesize $fileSize). Large files might require manual file upload instead of url submission." - } - info "$app`: Analysis in progress." - [PSCustomObject] @{ - 'App.Name' = $app - 'App.Url' = $url - 'App.Size' = filesize $fileSize - 'FileReport.Url' = $null - 'UrlReport.Url' = $url_report_url - } - return - } - - # EAFP: submission failed -> sleep, then retry - if (!$retrying) { - if (!$explained_rate_limit_sleeping) { - $explained_rate_limit_sleeping = $True - info "Sleeping 60+ seconds between requests due to VirusTotal's 4/min limit" - } - Start-Sleep -s (60 + $requests) - Submit-ToVirusTotal $url $app $do_scan $True - } else { - warn "$app`: VirusTotal submission of $url failed`:`n" + - "`tAPI returned $($result.StatusCode) after retrying" - } - } catch [Exception] { - warn "$app`: VirusTotal submission failed`: $($_.Exception.Message)" - return - } -} +$api_key = Get-VirusTotalApiKey +$scan = $opt.s -or $opt.scan $reports = $apps | ForEach-Object { $app = $_ @@ -275,107 +68,9 @@ $reports = $apps | ForEach-Object { return } - [int]$index = 0 - $urls = script:url $manifest $architecture - $urls | ForEach-Object { - $url = $_ - $index++ - if ($urls.GetType().IsArray) { - info "$app`: url $index" - } - $hash = hash_for_url $manifest $url $architecture - - try { - $isHashUnsupported = $false - if ($hash -match '(?[^:]+):(?.*)') { - $algo = $matches.algo - $hash = $matches.hash - if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { - $hash = $null - $isHashUnsupported = $true - warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." - } - } elseif ($hash) { - $algo = 'sha256' - } - if ($hash) { - $file_report = Get-VirusTotalResultByHash $hash $url $app - $file_report.'App.HashType' = $algo - $file_report - return - } elseif (!$isHashUnsupported) { - warn "$app`: Hash not found. Will search by url instead." - } - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - $file_report_not_found = $true - warn "$app`: File report not found. Will search by url instead." - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } - - try { - $url_report = Get-VirusTotalResultByUrl $url $app - $url_report.'App.Hash' = $hash - $url_report.'App.HashType' = $algo - if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { - if ($algo -eq 'sha256') { - if ($url_report.'UrlReport.Hash' -eq $hash) { - warn "$app`: Manual file upload is required (instead of url submission) for $url" - } else { - error "$app`: Hash not matched for $url" - } - } else { - error "$app`: Hash not matched or manual file upload is required (instead of url submission) for $url" - } - $url_report - return - } - if (!$url_report.'UrlReport.Hash') { - $url_report - return - } - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - warn "$app`: Url report not found. Will submit $url" - Submit-ToVirusTotal $url $app ($opt.scan -or $opt.s) - return - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } - - try { - $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app - $file_report.'App.Hash' = $hash - $file_report.'App.HashType' = $algo - $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' - $file_report - warn "$app`: Unable to check hash match for $url" - } catch [Exception] { - $exit_code = $exit_code -bor $_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - warn "$app`: File report not found for unknown reason. Manual file upload is required (instead of url submission)." - $url_report - } else { - if ($_.Exception.Response.StatusCode -in 204, 429) { - abort "$app`: VirusTotal request failed`: $($_.Exception.Message)" $exit_code - } - warn "$app`: VirusTotal request failed`: $($_.Exception.Message)" - return - } - } + $report = virustotal_check_app $app $manifest $architecture $api_key $scan + if ($report) { + $report } } if ($opt.p -or $opt.'passthru') { From daefa4a800d4514f6264d8a7dc7a97694523e712 Mon Sep 17 00:00:00 2001 From: felicijus Date: Wed, 15 Oct 2025 19:58:57 +0200 Subject: [PATCH 02/13] refactor(virustotal): extract virustotal check for one url --- lib/virustotal.ps1 | 181 +++++++++++++++++++++++---------------------- 1 file changed, 93 insertions(+), 88 deletions(-) diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index 6595e5a77f..f9f8ff4277 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -207,105 +207,110 @@ function Get-VirusTotalApiKey { return $api_key } -function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { - [int]$index = 0 - $urls = script:url $manifest $architecture - $urls | ForEach-Object { - $url = $_ - $index++ - if ($urls.GetType().IsArray) { - info "$app`: url $index" +function Check-VirusTotalUrl($app, $url, $hash, $api_key, $scan) { + $isHashUnsupported = $false + $algo = $null + + if ($hash -match '(?[^:]+):(?.*)') { + $algo = $matches.algo + $hash = $matches.hash + if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { + $hash = $null + $isHashUnsupported = $true + warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." } - $hash = hash_for_url $manifest $url $architecture + } elseif ($hash) { + $algo = 'sha256' + } - try { - $isHashUnsupported = $false - if ($hash -match '(?[^:]+):(?.*)') { - $algo = $matches.algo - $hash = $matches.hash - if ($matches.algo -inotin 'md5', 'sha1', 'sha256') { - $hash = $null - $isHashUnsupported = $true - warn "$app`: Unsupported hash $($matches.algo). Will search by url instead." - } - } elseif ($hash) { - $algo = 'sha256' - } - if ($hash) { - $file_report = Get-VirusTotalResultByHash $hash $url $app $api_key - $file_report.'App.HashType' = $algo - $file_report - return - } elseif (!$isHashUnsupported) { - warn "$app`: Hash not found. Will search by url instead." - } - } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - $file_report_not_found = $true - warn "$app`: File report not found. Will search by url instead." - } else { - warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" - if ($_.Exception.Response) { - warn "`tAPI returned $($_.Exception.Response.StatusCode)" - } - return + try { + if ($hash) { + $file_report = Get-VirusTotalResultByHash $hash $url $app $api_key + $file_report.'App.HashType' = $algo + return $file_report + } elseif (!$isHashUnsupported) { + warn "$app`: Hash not found. Will search by url instead." + } + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + $file_report_not_found = $true + warn "$app`: File report not found. Will search by url instead." + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" } + return } + } - try { - $url_report = Get-VirusTotalResultByUrl $url $app $api_key - $url_report.'App.Hash' = $hash - $url_report.'App.HashType' = $matches['algo'] - if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { - try { - $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key - if ($file_report.'FileReport.Hash' -ieq $matches['hash']) { - $file_report.'App.HashType' = $matches['algo'] - $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' - return $file_report - } - } catch { - warn "$app`: Unable to get file report for $($url_report.'UrlReport.Hash')" + try { + $url_report = Get-VirusTotalResultByUrl $url $app $api_key + $url_report.'App.Hash' = $hash + $url_report.'App.HashType' = $algo + if ($url_report.'UrlReport.Hash' -and ($file_report_not_found -eq $true) -and $hash) { + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + if ($file_report.'FileReport.Hash' -ieq $matches['hash']) { + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + return $file_report } + } catch { + warn "$app`: Unable to get file report for $($url_report.'UrlReport.Hash')" } - if (!$url_report.'UrlReport.Hash') { - Submit-ToVirusTotal $url $app $scan $api_key - return $url_report - } - } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - Submit-ToVirusTotal $url $app $scan $api_key - return - } else { - warn "$app`: VirusTotal URL report query failed`: $($_.Exception.Message)" - if ($_.Exception.Response) { - warn "`tAPI returned $($_.Exception.Response.StatusCode)" - } - return + } + if (!$url_report.'UrlReport.Hash') { + Submit-ToVirusTotal $url $app $scan $api_key + return $url_report + } + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + return + } else { + warn "$app`: VirusTotal URL report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" } + return } + } - try { - $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key - $file_report.'App.Hash' = $hash - $file_report.'App.HashType' = $matches['algo'] - $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' - $file_report - warn "$app`: Unable to check hash match for $url" - } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION - if ($_.Exception.Response.StatusCode -eq 404) { - Submit-ToVirusTotal $url $app $scan $api_key - $url_report - } else { - warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" - if ($_.Exception.Response) { - warn "`tAPI returned $($_.Exception.Response.StatusCode)" - } - return + try { + $file_report = Get-VirusTotalResultByHash $url_report.'UrlReport.Hash' $url $app $api_key + $file_report.'App.Hash' = $hash + $file_report.'App.HashType' = $algo + $file_report.'UrlReport.Url' = $url_report.'UrlReport.Url' + $file_report + warn "$app`: Unable to check hash match for $url" + } catch [Exception] { + $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + if ($_.Exception.Response.StatusCode -eq 404) { + Submit-ToVirusTotal $url $app $scan $api_key + $url_report + } else { + warn "$app`: VirusTotal file report query failed`: $($_.Exception.Message)" + if ($_.Exception.Response) { + warn "`tAPI returned $($_.Exception.Response.StatusCode)" } + return + } + } +} + +function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { + [int]$index = 0 + $urls = script:url $manifest $architecture + $urls | ForEach-Object { + $url = $_ + $index++ + if ($urls.GetType().IsArray) { + info "$app`: url $index" } + $hash = hash_for_url $manifest $url $architecture + Check-VirusTotalUrl $app $url $hash $api_key $scan } } From 08d22f23f6e33423d44d10075e0eec19f2e0d447 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 01:26:40 +0200 Subject: [PATCH 03/13] feat(virustotal): integrate VirusTotal checks into download and install processes --- lib/download.ps1 | 50 ++++++++++++++++++++++++++++++++------- lib/install.ps1 | 4 ++-- lib/virustotal.ps1 | 15 +++++++++++- libexec/scoop-config.ps1 | 3 +++ libexec/scoop-install.ps1 | 6 +++-- libexec/scoop-update.ps1 | 46 ++++++++++++++++++++++++++++++----- 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/lib/download.ps1 b/lib/download.ps1 index 70641cca7f..1bd50c8cff 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -1,8 +1,11 @@ # Description: Functions for downloading files +. "$PSScriptRoot\..\lib\core.ps1" +. "$PSScriptRoot\..\lib\virustotal.ps1" + ## Meta downloader -function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true) { +function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture, $dir, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { # we only want to show this warning once if (!$use_cache) { warn 'Cache is being ignored.' } @@ -12,11 +15,40 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture # can be multiple cookies: they will be used for all HTTP requests. $cookies = $manifest.cookie + # Pre-check all URLs with VirusTotal before downloading + $api_key = Get-VirusTotalApiKey + + $safe_urls = @() + if ($check_virustotal) { + foreach ($url in $urls) { + $hash = hash_for_url $manifest $url $architecture + $reports = Check-VirusTotalUrl $app $url $hash $api_key $false + $reports | ForEach-Object { + $file_report = $_ + $url = $file_report.'App.Url' + + $maliciousResults = $file_report.'FileReport.Malicious' + $suspiciousResults = $file_report.'FileReport.Suspicious' + if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) { + warn "$app`: One or more VirusTotal checks failed. Aborting before download." + } else { + info "$app`: Safe URL: $url" + $safe_urls += $url + } + } + } + if ($safe_urls.Count -eq 0) { + abort "No URL passed VirusTotal check for $app. Aborting before download." + } + } else { + $safe_urls = $urls + } + # download first if (Test-Aria2Enabled) { Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash } else { - foreach ($url in $urls) { + foreach ($url in $safe_urls) { $fname = url_filename $url try { @@ -45,7 +77,7 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture } } - return $urls.ForEach({ url_filename $_ }) + return $safe_urls.ForEach({ url_filename $_ }) } ## [System.Net] downloader @@ -457,9 +489,9 @@ function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $ warn "Download failed! (Error $lastexitcode) $(aria_exit_code $lastexitcode)" warn $urlstxt_content warn $aria2 - warn $(new_issue_msg $app $bucket "download via aria2 failed") + warn $(new_issue_msg $app $bucket 'download via aria2 failed') - Write-Host "Fallback to default downloader ..." + Write-Host 'Fallback to default downloader ...' try { foreach ($url in $urls) { @@ -666,7 +698,7 @@ function get_magic_bytes_pretty($file, $glue = ' ') { return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue } -Function Get-RemoteFileSize ($Uri) { +function Get-RemoteFileSize ($Uri) { $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing if (!$response.Headers.StatusCode) { $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } @@ -689,13 +721,13 @@ function url_remote_filename($url) { # this function extracts the original filename from the URL. $uri = (New-Object URI $url) $basename = Split-Path $uri.PathAndQuery -Leaf - If ($basename -match '.*[?=]+([\w._-]+)') { + if ($basename -match '.*[?=]+([\w._-]+)') { $basename = $matches[1] } - If (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { + if (($basename -notlike '*.*') -or ($basename -match '^[v.\d]+$')) { $basename = Split-Path $uri.AbsolutePath -Leaf } - If (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { + if (($basename -notlike '*.*') -and ($uri.Fragment -ne '')) { $basename = $uri.Fragment.Trim('/', '#') } return $basename diff --git a/lib/install.ps1 b/lib/install.ps1 index 6125697967..f2edd1fcd7 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -5,7 +5,7 @@ function nightly_version($quiet = $false) { return "nightly-$(Get-Date -Format 'yyyyMMdd')" } -function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true) { +function install_app($app, $architecture, $global, $suggested, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $app, $manifest, $bucket, $url = Get-Manifest $app if (!$manifest) { @@ -49,7 +49,7 @@ function install_app($app, $architecture, $global, $suggested, $use_cache = $tru $original_dir = $dir # keep reference to real (not linked) directory $persist_dir = persistdir $app $global - $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash + $fname = Invoke-ScoopDownload $app $version $manifest $bucket $architecture $dir $use_cache $check_hash $check_virustotal Invoke-Extraction -Path $dir -Name $fname -Manifest $manifest -ProcessorArchitecture $architecture Invoke-HookScript -HookType 'pre_install' -Manifest $manifest -ProcessorArchitecture $architecture diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index f9f8ff4277..0eb7e0d872 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -1,5 +1,5 @@ . "$PSScriptRoot\json.ps1" # 'json_path' -. "$PSScriptRoot\download.ps1" # 'hash_for_url' +# . "$PSScriptRoot\download.ps1" # 'hash_for_url' # Error codes $script:_ERR_UNSAFE = 2 @@ -314,3 +314,16 @@ function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { Check-VirusTotalUrl $app $url $hash $api_key $scan } } + +function hash_for_url($manifest, $url, $arch) { + $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } + + if ($hashes.length -eq 0) { return $null } + + $urls = @(script:url $manifest $arch) + + $index = [array]::IndexOf($urls, $url) + if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } + + @($hashes)[$index] +} diff --git a/libexec/scoop-config.ps1 b/libexec/scoop-config.ps1 index 6007bd6434..6f14a04335 100644 --- a/libexec/scoop-config.ps1 +++ b/libexec/scoop-config.ps1 @@ -92,6 +92,9 @@ # API key used for uploading/scanning files using virustotal. # See: 'https://support.virustotal.com/hc/en-us/articles/115002088769-Please-give-me-an-API-key' # +# use_virustotal: $true|$false +# When set to $true, Scoop will always use VirusTotal to scan files after downloading. +# # cat_style: # When set to a non-empty string, Scoop will use 'bat' to display the manifest for # the `scoop cat` command and while doing manifest review. This requires 'bat' to be diff --git a/libexec/scoop-install.ps1 b/libexec/scoop-install.ps1 index 6af199a8b8..dab7f0941a 100644 --- a/libexec/scoop-install.ps1 +++ b/libexec/scoop-install.ps1 @@ -24,6 +24,7 @@ # -i, --independent Don't install dependencies automatically # -k, --no-cache Don't use the download cache # -s, --skip-hash-check Skip hash validation (use with caution!) +# -w, --virustotal-check Check the download against VirusTotal (may be slow) # -u, --no-update-scoop Don't update Scoop before installing if it's outdated # -a, --arch <32bit|64bit|arm64> Use the specified architecture, if the app supports it @@ -43,11 +44,12 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'giksua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'no-update-scoop', 'arch=' +$opt, $apps, $err = getopt $args 'gikswua:' 'global', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'no-update-scoop', 'arch=' if ($err) { error "scoop install: $err"; exit 1 } $global = $opt.g -or $opt.global $check_hash = !($opt.s -or $opt.'skip-hash-check') +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) $independent = $opt.i -or $opt.independent $use_cache = !($opt.k -or $opt.'no-cache') $architecture = Get-DefaultArchitecture @@ -132,7 +134,7 @@ if ((Test-Aria2Enabled) -and (get_config 'aria2-warning-enabled' $true)) { warn "Should it cause issues, run 'scoop config aria2-enabled false' to disable it." warn "To disable this warning, run 'scoop config aria2-warning-enabled false'." } -$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash } +$apps | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal } show_suggestions $suggested diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index bc590a13f6..1eedd182ec 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -11,6 +11,7 @@ # -i, --independent Don't install dependencies automatically # -k, --no-cache Don't use the download cache # -s, --skip-hash-check Skip hash validation (use with caution!) +# -w, --virustotal-check Check the download against VirusTotal (may be slow) # -q, --quiet Hide extraneous messages # -a, --all Update all apps (alternative to '*') @@ -29,11 +30,12 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'gfiksqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'quiet', 'all' +$opt, $apps, $err = getopt $args 'gfikswqa' 'global', 'force', 'independent', 'no-cache', 'skip-hash-check', 'virustotal-check', 'quiet', 'all' if ($err) { error "scoop update: $err"; exit 1 } $global = $opt.g -or $opt.global $force = $opt.f -or $opt.force $check_hash = !($opt.s -or $opt.'skip-hash-check') +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) $use_cache = !($opt.k -or $opt.'no-cache') $quiet = $opt.q -or $opt.quiet $independent = $opt.i -or $opt.independent @@ -258,7 +260,7 @@ function Sync-Bucket { } } -function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true) { +function update($app, $global, $quiet = $false, $independent, $suggested, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $old_version = Select-CurrentVersion -AppName $app -Global:$global $old_manifest = installed_manifest $app $old_version $global $install = install_info $app $old_version $global @@ -300,6 +302,38 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c } #endregion Workaround for #2952 + # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work + $urls = @(script:url $manifest $architecture) + + # Pre-check all URLs with VirusTotal before downloading + $api_key = Get-VirusTotalApiKey + + $safe_urls = @() + if ($check_virustotal) { + foreach ($url in $urls) { + $hash = hash_for_url $manifest $url $architecture + $reports = Check-VirusTotalUrl $app $url $hash $api_key $false + $reports | ForEach-Object { + $file_report = $_ + $url = $file_report.'App.Url' + + $maliciousResults = $file_report.'FileReport.Malicious' + $suspiciousResults = $file_report.'FileReport.Suspicious' + if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) { + warn "$app`: One or more VirusTotal checks failed. Aborting before download." + } else { + info "$app`: Safe URL: $url" + $safe_urls += $url + } + } + } + if ($safe_urls.Count -eq 0) { + abort "No URL passed VirusTotal check for $app. Aborting before download." + } + } else { + $safe_urls = $urls + } + # region Workaround # Workaround for https://github.com/ScoopInstaller/Scoop/issues/2220 until install is refactored # Remove and replace whole region after proper fix @@ -309,7 +343,7 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c } else { $urls = script:url $manifest $architecture - foreach ($url in $urls) { + foreach ($url in $safe_urls) { Invoke-CachedDownload $app $version $url $null $manifest.cookie $true if ($check_hash) { @@ -376,12 +410,12 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c } if ($independent) { - install_app $app $architecture $global $suggested $use_cache $check_hash + install_app $app $architecture $global $suggested $use_cache $check_hash $check_virustotal } else { # Also add missing dependencies $apps = @(Get-Dependency $app $architecture) -ne $app ensure_none_failed $apps - $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash } + $apps.Where({ !(installed $_) }) + $app | ForEach-Object { install_app $_ $architecture $global $suggested $use_cache $check_hash $check_virustotal} } } @@ -462,7 +496,7 @@ if (-not ($apps -or $all)) { $suggested = @{} # $outdated is a list of ($app, $global) tuples - $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash } + $outdated | ForEach-Object { update @_ $quiet $independent $suggested $use_cache $check_hash $check_virustotal } } exit 0 From 934f9bd955565da6f2c36371507961108d362dc8 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 20:28:13 +0200 Subject: [PATCH 04/13] refactor(download): move hash-related functions to seperate lib file, update imports --- bin/checkhashes.ps1 | 1 + lib/autoupdate.ps1 | 2 ++ lib/download.ps1 | 30 +----------------------------- lib/hash.ps1 | 28 ++++++++++++++++++++++++++++ lib/install.ps1 | 1 + lib/virustotal.ps1 | 15 +-------------- libexec/scoop-download.ps1 | 1 + libexec/scoop-update.ps1 | 1 + 8 files changed, 36 insertions(+), 43 deletions(-) create mode 100644 lib/hash.ps1 diff --git a/bin/checkhashes.ps1 b/bin/checkhashes.ps1 index 6e6420fb2a..a1eff6b164 100644 --- a/bin/checkhashes.ps1 +++ b/bin/checkhashes.ps1 @@ -42,6 +42,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" +. "$PSScriptRoot\..\lib\hash.ps1" # 'get_hash' . "$PSScriptRoot\..\lib\buckets.ps1" . "$PSScriptRoot\..\lib\autoupdate.ps1" . "$PSScriptRoot\..\lib\json.ps1" diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index fbc3ebfdee..d05d04be46 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,5 +1,7 @@ # Must included with 'json.ps1' +. "$PSScriptRoot\..\lib\hash.ps1" # 'get_hash' + function format_hash([String] $hash) { $hash = $hash.toLower() diff --git a/lib/download.ps1 b/lib/download.ps1 index 1bd50c8cff..92e79ff32b 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -1,6 +1,7 @@ # Description: Functions for downloading files . "$PSScriptRoot\..\lib\core.ps1" +. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\virustotal.ps1" ## Meta downloader @@ -733,21 +734,6 @@ function url_remote_filename($url) { return $basename } -### Hash-related functions - -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::IndexOf($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] -} - function check_hash($file, $hash, $app_name) { # returns (ok, err) if (!$hash) { @@ -783,19 +769,5 @@ function check_hash($file, $hash, $app_name) { return $true, $null } -function get_hash([String] $multihash) { - $type, $hash = $multihash -split ':' - if (!$hash) { - # no type specified, assume sha256 - $type, $hash = 'sha256', $multihash - } - - if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { - return $null, "Hash type '$type' isn't supported." - } - - return $type, $hash.ToLower() -} - # Setup proxy globally setup_proxy diff --git a/lib/hash.ps1 b/lib/hash.ps1 new file mode 100644 index 0000000000..4cb4826436 --- /dev/null +++ b/lib/hash.ps1 @@ -0,0 +1,28 @@ +### Hash-related functions + +function hash_for_url($manifest, $url, $arch) { + $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } + + if ($hashes.length -eq 0) { return $null } + + $urls = @(script:url $manifest $arch) + + $index = [array]::IndexOf($urls, $url) + if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } + + @($hashes)[$index] +} + +function get_hash([String] $multihash) { + $type, $hash = $multihash -split ':' + if (!$hash) { + # no type specified, assume sha256 + $type, $hash = 'sha256', $multihash + } + + if (@('md5', 'sha1', 'sha256', 'sha512') -notcontains $type) { + return $null, "Hash type '$type' isn't supported." + } + + return $type, $hash.ToLower() +} diff --git a/lib/install.ps1 b/lib/install.ps1 index f2edd1fcd7..9b1b688e7d 100644 --- a/lib/install.ps1 +++ b/lib/install.ps1 @@ -1,3 +1,4 @@ + function nightly_version($quiet = $false) { if (!$quiet) { warn "This is a nightly version. Downloaded files won't be verified." diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index 0eb7e0d872..7531e4c0dc 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -1,5 +1,5 @@ . "$PSScriptRoot\json.ps1" # 'json_path' -# . "$PSScriptRoot\download.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' # Error codes $script:_ERR_UNSAFE = 2 @@ -314,16 +314,3 @@ function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { Check-VirusTotalUrl $app $url $hash $api_key $scan } } - -function hash_for_url($manifest, $url, $arch) { - $hashes = @(hash $manifest $arch) | Where-Object { $_ -ne $null } - - if ($hashes.length -eq 0) { return $null } - - $urls = @(script:url $manifest $arch) - - $index = [array]::IndexOf($urls, $url) - if ($index -eq -1) { abort "Couldn't find hash in manifest for '$url'." } - - @($hashes)[$index] -} diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index 34a78ea952..293ea14f55 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -21,6 +21,7 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) +. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\autoupdate.ps1" # 'generate_user_manifest' (indirectly) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index 1eedd182ec..1b6c2bbb4d 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -22,6 +22,7 @@ . "$PSScriptRoot\..\lib\psmodules.ps1" . "$PSScriptRoot\..\lib\decompress.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" +. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\versions.ps1" . "$PSScriptRoot\..\lib\depends.ps1" . "$PSScriptRoot\..\lib\install.ps1" From 81531dee22114fd6171a49802237684f2a3ca765 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 20:39:28 +0200 Subject: [PATCH 05/13] feat(CHANGELOG): update to include VirusTotal refactor and pre-download checks --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 920ec82676..fc36b39ac8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ - **scoop-uninstall**: Allow access to `$bucket` in uninstall scripts ([#6380](https://github.com/ScoopInstaller/Scoop/issues/6380)) - **install:** Add separator at the end of notes, highlight suggestions ([#6418](https://github.com/ScoopInstaller/Scoop/issues/6418)) +- **virustotal:** Refactor into lib and integrate pre-download checks for install and update ([#6525](https://github.com/ScoopInstaller/Scoop/issues/6525)) + ### Bug Fixes From e8003acea10e12539ee21ba1bcc33ba029a2b873 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 21:01:50 +0200 Subject: [PATCH 06/13] refactor(download): move API key retrieval inside VirusTotal check --- lib/download.ps1 | 3 +-- libexec/scoop-update.ps1 | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/download.ps1 b/lib/download.ps1 index 92e79ff32b..51e65c7222 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -17,10 +17,9 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture $cookies = $manifest.cookie # Pre-check all URLs with VirusTotal before downloading - $api_key = Get-VirusTotalApiKey - $safe_urls = @() if ($check_virustotal) { + $api_key = Get-VirusTotalApiKey foreach ($url in $urls) { $hash = hash_for_url $manifest $url $architecture $reports = Check-VirusTotalUrl $app $url $hash $api_key $false diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index 1b6c2bbb4d..bb69d52f5c 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -307,10 +307,9 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c $urls = @(script:url $manifest $architecture) # Pre-check all URLs with VirusTotal before downloading - $api_key = Get-VirusTotalApiKey - $safe_urls = @() if ($check_virustotal) { + $api_key = Get-VirusTotalApiKey foreach ($url in $urls) { $hash = hash_for_url $manifest $url $architecture $reports = Check-VirusTotalUrl $app $url $hash $api_key $false From f143dad4d247c275caa9d2b51bd85c177e8e90b5 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 22:17:10 +0200 Subject: [PATCH 07/13] feat(virustotal): integrate VirusTotal URL checks into Aria2 download --- lib/download.ps1 | 47 ++++++++++++-------------------------- lib/virustotal.ps1 | 32 ++++++++++++++++++++++++++ libexec/scoop-download.ps1 | 14 +++++++++--- libexec/scoop-update.ps1 | 37 ++++++------------------------ 4 files changed, 65 insertions(+), 65 deletions(-) diff --git a/lib/download.ps1 b/lib/download.ps1 index 51e65c7222..5108cc28b2 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -16,39 +16,17 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture # can be multiple cookies: they will be used for all HTTP requests. $cookies = $manifest.cookie - # Pre-check all URLs with VirusTotal before downloading - $safe_urls = @() - if ($check_virustotal) { - $api_key = Get-VirusTotalApiKey - foreach ($url in $urls) { - $hash = hash_for_url $manifest $url $architecture - $reports = Check-VirusTotalUrl $app $url $hash $api_key $false - $reports | ForEach-Object { - $file_report = $_ - $url = $file_report.'App.Url' - - $maliciousResults = $file_report.'FileReport.Malicious' - $suspiciousResults = $file_report.'FileReport.Suspicious' - if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) { - warn "$app`: One or more VirusTotal checks failed. Aborting before download." - } else { - info "$app`: Safe URL: $url" - $safe_urls += $url - } - } - } - if ($safe_urls.Count -eq 0) { - abort "No URL passed VirusTotal check for $app. Aborting before download." - } - } else { - $safe_urls = $urls - } - # download first if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $dir $cookies $use_cache $check_hash $check_virustotal } else { - foreach ($url in $safe_urls) { + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } + + foreach ($url in $urls) { $fname = url_filename $url try { @@ -77,7 +55,7 @@ function Invoke-ScoopDownload ($app, $version, $manifest, $bucket, $architecture } } - return $safe_urls.ForEach({ url_filename $_ }) + return $urls.ForEach({ url_filename $_ }) } ## [System.Net] downloader @@ -361,9 +339,14 @@ function get_filename_from_metalink($file) { return $filename } -function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $dir, $cookies = $null, $use_cache = $true, $check_hash = $true) { +function Invoke-CachedAria2Download ($app, $version, $manifest, $architecture, $dir, $cookies = $null, $use_cache = $true, $check_hash = $true, $check_virustotal = $false) { $data = @{} $urls = @(script:url $manifest $architecture) + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } # aria2 input file $urlstxt = Join-Path $cachedir "$app.txt" diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index 7531e4c0dc..2d7b5f5ed6 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -314,3 +314,35 @@ function virustotal_check_app($app, $manifest, $architecture, $api_key, $scan) { Check-VirusTotalUrl $app $url $hash $api_key $scan } } + +# return only the URLs that passed VirusTotal checks +function Test-UrlsWithVirusTotal($app, $urls, $manifest, $architecture) { + $safe_urls = @() + $api_key = Get-VirusTotalApiKey + + foreach ($url in $urls) { + $hash = hash_for_url $manifest $url $architecture + $reports = Check-VirusTotalUrl $app $url $hash $api_key $false + + $reports | ForEach-Object { + $file_report = $_ + $url = $file_report.'App.Url' + + $maliciousResults = $file_report.'FileReport.Malicious' + $suspiciousResults = $file_report.'FileReport.Suspicious' + + if ($maliciousResults -eq 0 -and $suspiciousResults -eq 0) { + info "$app`: Safe URL: $url" + $safe_urls += $url + } else { + warn "$app`: One or more VirusTotal checks failed. Aborting before download." + } + } + } + + if ($safe_urls.Count -eq 0) { + abort "No URL passed VirusTotal check for $app. Aborting before download." + } + + return $safe_urls +} diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index 293ea14f55..f4a8ec381c 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -18,6 +18,7 @@ # -s, --skip-hash-check Skip hash verification (use with caution!) # -u, --no-update-scoop Don't update Scoop before downloading if it's outdated # -a, --arch <32bit|64bit|arm64> Use the specified architecture, if the app supports it +# -w, --virustotal-check Check the download against VirusTotal (may be slow) . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) @@ -31,12 +32,13 @@ if (get_config USE_SQLITE_CACHE) { . "$PSScriptRoot\..\lib\database.ps1" } -$opt, $apps, $err = getopt $args 'fsua:' 'force', 'skip-hash-check', 'no-update-scoop', 'arch=' +$opt, $apps, $err = getopt $args 'fsua:w' 'force', 'skip-hash-check', 'no-update-scoop', 'arch=', 'virustotal-check' if ($err) { error "scoop download: $err"; exit 1 } $check_hash = !($opt.s -or $opt.'skip-hash-check') $use_cache = !($opt.f -or $opt.force) $architecture = Get-DefaultArchitecture +$check_virustotal = $opt.w -or $opt.'virustotal-check' -or (get_config USE_VIRUSTOTAL $false) try { $architecture = Format-ArchitectureString ($opt.a + $opt.arch) } catch { @@ -102,9 +104,15 @@ foreach ($curr_app in $apps) { } if(Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $use_cache $curr_check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $use_cache $curr_check_hash $check_virustotal } else { - foreach($url in script:url $manifest $architecture) { + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } + + foreach($url in $urls) { try { Invoke-CachedDownload $app $version $url $null $manifest.cookie $use_cache } catch { diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index bb69d52f5c..4885914626 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -306,44 +306,21 @@ function update($app, $global, $quiet = $false, $independent, $suggested, $use_c # can be multiple urls: if there are, then installer should go first to make 'installer.args' section work $urls = @(script:url $manifest $architecture) - # Pre-check all URLs with VirusTotal before downloading - $safe_urls = @() - if ($check_virustotal) { - $api_key = Get-VirusTotalApiKey - foreach ($url in $urls) { - $hash = hash_for_url $manifest $url $architecture - $reports = Check-VirusTotalUrl $app $url $hash $api_key $false - $reports | ForEach-Object { - $file_report = $_ - $url = $file_report.'App.Url' - - $maliciousResults = $file_report.'FileReport.Malicious' - $suspiciousResults = $file_report.'FileReport.Suspicious' - if ($maliciousResults -gt 0 -or $suspiciousResults -gt 0) { - warn "$app`: One or more VirusTotal checks failed. Aborting before download." - } else { - info "$app`: Safe URL: $url" - $safe_urls += $url - } - } - } - if ($safe_urls.Count -eq 0) { - abort "No URL passed VirusTotal check for $app. Aborting before download." - } - } else { - $safe_urls = $urls - } - # region Workaround # Workaround for https://github.com/ScoopInstaller/Scoop/issues/2220 until install is refactored # Remove and replace whole region after proper fix Write-Host 'Downloading new version' if (Test-Aria2Enabled) { - Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $true $check_hash + Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $true $check_hash $check_virustotal } else { $urls = script:url $manifest $architecture + $urls = if ($check_virustotal) { + Test-UrlsWithVirusTotal $app $urls $manifest $architecture + } else { + $urls + } - foreach ($url in $safe_urls) { + foreach ($url in $urls) { Invoke-CachedDownload $app $version $url $null $manifest.cookie $true if ($check_hash) { From 15d99e777473c9cd3519ef30332227bb3cfe0634 Mon Sep 17 00:00:00 2001 From: felicijus Date: Thu, 16 Oct 2025 22:26:15 +0200 Subject: [PATCH 08/13] fix(config): update VirusTotal usage description to reflect pre-download scanning --- libexec/scoop-config.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libexec/scoop-config.ps1 b/libexec/scoop-config.ps1 index 6f14a04335..a420bf6a26 100644 --- a/libexec/scoop-config.ps1 +++ b/libexec/scoop-config.ps1 @@ -93,7 +93,7 @@ # See: 'https://support.virustotal.com/hc/en-us/articles/115002088769-Please-give-me-an-API-key' # # use_virustotal: $true|$false -# When set to $true, Scoop will always use VirusTotal to scan files after downloading. +# When set to $true, Scoop will always use VirusTotal to scan files before downloading. # # cat_style: # When set to a non-empty string, Scoop will use 'bat' to display the manifest for From 4b9e3f1b2cc26a23b57643fc059f3eb00dcbd1d4 Mon Sep 17 00:00:00 2001 From: felicijus Date: Sun, 19 Oct 2025 15:23:42 +0200 Subject: [PATCH 09/13] fix(virustotal): update report structure to include malicious and suspicious counts, undetected, unsafe and total --- lib/virustotal.ps1 | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index 2d7b5f5ed6..8981967f78 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -73,10 +73,14 @@ function Get-VirusTotalResultByHash ($hash, $url, $app, $api_key) { 'App.Size' = filesize $fileSize 'FileReport.Url' = $report_url 'FileReport.Hash' = $report_hash - 'FileReport.Malicious' = if ($maliciousResults) { $maliciousResults } else { 0 } - 'FileReport.Suspicious' = if ($suspiciousResults) { $suspiciousResults } else { 0 } + 'FileReport.MaliciousResults' = if ($maliciousResults) { $maliciousResults } else { @() } + 'FileReport.SuspiciousResults' = if ($suspiciousResults) { $suspiciousResults } else { @() } + 'FileReport.Malicious' = $malicious + 'FileReport.Suspicious' = $suspicious 'FileReport.Timeout' = $timeout 'FileReport.Undetected' = $undetected + 'FileReport.Unsafe' = $unsafe + 'FileReort.Total' = $total 'UrlReport.Url' = $null } } @@ -328,20 +332,17 @@ function Test-UrlsWithVirusTotal($app, $urls, $manifest, $architecture) { $file_report = $_ $url = $file_report.'App.Url' - $maliciousResults = $file_report.'FileReport.Malicious' - $suspiciousResults = $file_report.'FileReport.Suspicious' - - if ($maliciousResults -eq 0 -and $suspiciousResults -eq 0) { + if ($file_report.'FileReport.Unsafe' -eq 0) { info "$app`: Safe URL: $url" $safe_urls += $url } else { - warn "$app`: One or more VirusTotal checks failed. Aborting before download." + warn "$app`: Unsafe URL: $url" } } } - if ($safe_urls.Count -eq 0) { - abort "No URL passed VirusTotal check for $app. Aborting before download." + if ($safe_urls.Count -ne $urls.Count) { + abort "VirusTotal check for $app failed. Aborting before download." } return $safe_urls From 9784acd506ca43b8988641f4354a8385149e24f3 Mon Sep 17 00:00:00 2001 From: felicijus Date: Sun, 19 Oct 2025 15:31:30 +0200 Subject: [PATCH 10/13] fix(virustotal): 'FileReort.Total' should be 'FileReport.Total' (missing 'p' in 'Report') --- lib/virustotal.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index 8981967f78..e580d0ff24 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -80,7 +80,7 @@ function Get-VirusTotalResultByHash ($hash, $url, $app, $api_key) { 'FileReport.Timeout' = $timeout 'FileReport.Undetected' = $undetected 'FileReport.Unsafe' = $unsafe - 'FileReort.Total' = $total + 'FileReport.Total' = $total 'UrlReport.Url' = $null } } From e87839e5b5e5a2b93963c533fb9071f23300c186 Mon Sep 17 00:00:00 2001 From: felicijus Date: Sun, 19 Oct 2025 18:38:27 +0200 Subject: [PATCH 11/13] refactor(helper): create helper folder for scripts with helper functions --- bin/checkhashes.ps1 | 2 +- lib/autoupdate.ps1 | 2 +- lib/download.ps1 | 9 ++------- lib/helper/file-information.ps1 | 8 ++++++++ lib/{ => helper}/hash.ps1 | 0 lib/virustotal.ps1 | 3 ++- libexec/scoop-download.ps1 | 2 +- libexec/scoop-info.ps1 | 1 + libexec/scoop-update.ps1 | 2 +- 9 files changed, 17 insertions(+), 12 deletions(-) create mode 100644 lib/helper/file-information.ps1 rename lib/{ => helper}/hash.ps1 (100%) diff --git a/bin/checkhashes.ps1 b/bin/checkhashes.ps1 index a1eff6b164..45ab735a2d 100644 --- a/bin/checkhashes.ps1 +++ b/bin/checkhashes.ps1 @@ -42,7 +42,7 @@ param( . "$PSScriptRoot\..\lib\core.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" -. "$PSScriptRoot\..\lib\hash.ps1" # 'get_hash' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'get_hash' . "$PSScriptRoot\..\lib\buckets.ps1" . "$PSScriptRoot\..\lib\autoupdate.ps1" . "$PSScriptRoot\..\lib\json.ps1" diff --git a/lib/autoupdate.ps1 b/lib/autoupdate.ps1 index d05d04be46..8b05d91ca3 100644 --- a/lib/autoupdate.ps1 +++ b/lib/autoupdate.ps1 @@ -1,6 +1,6 @@ # Must included with 'json.ps1' -. "$PSScriptRoot\..\lib\hash.ps1" # 'get_hash' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'get_hash' function format_hash([String] $hash) { $hash = $hash.toLower() diff --git a/lib/download.ps1 b/lib/download.ps1 index 5108cc28b2..1da0090e2d 100644 --- a/lib/download.ps1 +++ b/lib/download.ps1 @@ -1,7 +1,8 @@ # Description: Functions for downloading files . "$PSScriptRoot\..\lib\core.ps1" -. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' . "$PSScriptRoot\..\lib\virustotal.ps1" ## Meta downloader @@ -681,12 +682,6 @@ function get_magic_bytes_pretty($file, $glue = ' ') { return (get_magic_bytes $file | ForEach-Object { $_.ToString('x2') }) -join $glue } -function Get-RemoteFileSize ($Uri) { - $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing - if (!$response.Headers.StatusCode) { - $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } - } -} function ftp_file_size($url) { $request = [net.ftpwebrequest]::create($url) diff --git a/lib/helper/file-information.ps1 b/lib/helper/file-information.ps1 new file mode 100644 index 0000000000..379d8204a4 --- /dev/null +++ b/lib/helper/file-information.ps1 @@ -0,0 +1,8 @@ +### Remote file information + +function Get-RemoteFileSize ($Uri) { + $response = Invoke-WebRequest -Uri $Uri -Method HEAD -UseBasicParsing + if (!$response.Headers.StatusCode) { + $response.Headers.'Content-Length' | ForEach-Object { [int]$_ } + } +} diff --git a/lib/hash.ps1 b/lib/helper/hash.ps1 similarity index 100% rename from lib/hash.ps1 rename to lib/helper/hash.ps1 diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index e580d0ff24..e20134a6d9 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -1,5 +1,6 @@ . "$PSScriptRoot\json.ps1" # 'json_path' -. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' # Error codes $script:_ERR_UNSAFE = 2 diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index f4a8ec381c..a86158ea2c 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -22,7 +22,7 @@ . "$PSScriptRoot\..\lib\getopt.ps1" . "$PSScriptRoot\..\lib\json.ps1" # 'autoupdate.ps1' (indirectly) -. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\autoupdate.ps1" # 'generate_user_manifest' (indirectly) . "$PSScriptRoot\..\lib\versions.ps1" # 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\manifest.ps1" # 'generate_user_manifest' 'Get-Manifest' diff --git a/libexec/scoop-info.ps1 b/libexec/scoop-info.ps1 index 1c147e17c0..f394dc7554 100644 --- a/libexec/scoop-info.ps1 +++ b/libexec/scoop-info.ps1 @@ -7,6 +7,7 @@ . "$PSScriptRoot\..\lib\manifest.ps1" # 'Get-Manifest' . "$PSScriptRoot\..\lib\versions.ps1" # 'Get-InstalledVersion', 'Select-CurrentVersion' . "$PSScriptRoot\..\lib\download.ps1" # 'Get-RemoteFileSize' +. "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' $opt, $app, $err = getopt $args 'v' 'verbose' $original_app = $app diff --git a/libexec/scoop-update.ps1 b/libexec/scoop-update.ps1 index 4885914626..51f43218c6 100644 --- a/libexec/scoop-update.ps1 +++ b/libexec/scoop-update.ps1 @@ -22,7 +22,7 @@ . "$PSScriptRoot\..\lib\psmodules.ps1" . "$PSScriptRoot\..\lib\decompress.ps1" . "$PSScriptRoot\..\lib\manifest.ps1" -. "$PSScriptRoot\..\lib\hash.ps1" # 'hash_for_url' +. "$PSScriptRoot\..\lib\helper\hash.ps1" # 'hash_for_url' . "$PSScriptRoot\..\lib\versions.ps1" . "$PSScriptRoot\..\lib\depends.ps1" . "$PSScriptRoot\..\lib\install.ps1" From ee8d875cd0ef2d5298c82811f92040dfb099bf6f Mon Sep 17 00:00:00 2001 From: felicijus Date: Sun, 19 Oct 2025 19:05:50 +0200 Subject: [PATCH 12/13] fix(scoop-download): get urls first --- libexec/scoop-download.ps1 | 1 + 1 file changed, 1 insertion(+) diff --git a/libexec/scoop-download.ps1 b/libexec/scoop-download.ps1 index a86158ea2c..7db9bb3c27 100644 --- a/libexec/scoop-download.ps1 +++ b/libexec/scoop-download.ps1 @@ -106,6 +106,7 @@ foreach ($curr_app in $apps) { if(Test-Aria2Enabled) { Invoke-CachedAria2Download $app $version $manifest $architecture $cachedir $manifest.cookie $use_cache $curr_check_hash $check_virustotal } else { + $urls = @(script:url $manifest $architecture) $urls = if ($check_virustotal) { Test-UrlsWithVirusTotal $app $urls $manifest $architecture } else { From ba8bb50a40e9990bb41e9dabc04ed2071b3d9d9e Mon Sep 17 00:00:00 2001 From: felicijus Date: Sun, 19 Oct 2025 19:17:39 +0200 Subject: [PATCH 13/13] fix(virustotal): common exit_code handling --- lib/virustotal.ps1 | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lib/virustotal.ps1 b/lib/virustotal.ps1 index e20134a6d9..8cca736909 100644 --- a/lib/virustotal.ps1 +++ b/lib/virustotal.ps1 @@ -3,15 +3,15 @@ . "$PSScriptRoot\..\lib\helper\file-information.ps1" # 'Get-RemoteFileSize' # Error codes -$script:_ERR_UNSAFE = 2 -$script:_ERR_EXCEPTION = 4 -$script:_ERR_NO_INFO = 8 -$script:_ERR_NO_API_KEY = 16 +$_ERR_UNSAFE = 2 +$_ERR_EXCEPTION = 4 +$_ERR_NO_INFO = 8 +$_ERR_NO_API_KEY = 16 # Global state variables $script:requests = 0 $script:explained_rate_limit_sleeping = $False -$script:exit_code = 0 +$exit_code = 0 function ConvertTo-VirusTotalUrlId ($url) { $url_id = [System.Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($url)) @@ -86,7 +86,7 @@ function Get-VirusTotalResultByHash ($hash, $url, $app, $api_key) { } } if ($unsafe -gt 0) { - $Script:exit_code = $exit_code -bor $script:_ERR_UNSAFE + $exit_code = $exit_code -bor $_ERR_UNSAFE } } @@ -237,7 +237,7 @@ function Check-VirusTotalUrl($app, $url, $hash, $api_key, $scan) { warn "$app`: Hash not found. Will search by url instead." } } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { $file_report_not_found = $true warn "$app`: File report not found. Will search by url instead." @@ -271,7 +271,7 @@ function Check-VirusTotalUrl($app, $url, $hash, $api_key, $scan) { return $url_report } } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { Submit-ToVirusTotal $url $app $scan $api_key return @@ -292,7 +292,7 @@ function Check-VirusTotalUrl($app, $url, $hash, $api_key, $scan) { $file_report warn "$app`: Unable to check hash match for $url" } catch [Exception] { - $script:exit_code = $exit_code -bor $script:_ERR_EXCEPTION + $exit_code = $exit_code -bor $_ERR_EXCEPTION if ($_.Exception.Response.StatusCode -eq 404) { Submit-ToVirusTotal $url $app $scan $api_key $url_report @@ -342,7 +342,7 @@ function Test-UrlsWithVirusTotal($app, $urls, $manifest, $architecture) { } } - if ($safe_urls.Count -ne $urls.Count) { + if ($safe_urls.Count -eq 0) { abort "VirusTotal check for $app failed. Aborting before download." }