diff --git a/api/cache/getLogStores.js b/api/cache/getLogStores.js index 3089192196d9..ae9b5db64672 100644 --- a/api/cache/getLogStores.js +++ b/api/cache/getLogStores.js @@ -44,6 +44,7 @@ const namespaces = { [CacheKeys.TOKEN_CONFIG]: standardCache(CacheKeys.TOKEN_CONFIG, Time.THIRTY_MINUTES), [CacheKeys.GEN_TITLE]: standardCache(CacheKeys.GEN_TITLE, Time.TWO_MINUTES), [CacheKeys.S3_EXPIRY_INTERVAL]: standardCache(CacheKeys.S3_EXPIRY_INTERVAL, Time.THIRTY_MINUTES), + [CacheKeys.AZURE_EXPIRY_INTERVAL]: standardCache(CacheKeys.AZURE_EXPIRY_INTERVAL, Time.THIRTY_MINUTES), [CacheKeys.MODEL_QUERIES]: standardCache(CacheKeys.MODEL_QUERIES), [CacheKeys.AUDIO_RUNS]: standardCache(CacheKeys.AUDIO_RUNS, Time.TEN_MINUTES), [CacheKeys.MESSAGES]: standardCache(CacheKeys.MESSAGES, Time.ONE_MINUTE), diff --git a/api/server/controllers/UserController.js b/api/server/controllers/UserController.js index 7a9dd8125e9e..364375b40280 100644 --- a/api/server/controllers/UserController.js +++ b/api/server/controllers/UserController.js @@ -38,6 +38,17 @@ const { verifyEmail, resendVerificationEmail } = require('~/server/services/Auth const { getMCPManager, getFlowStateManager, getMCPServersRegistry } = require('~/config'); const { invalidateCachedTools } = require('~/server/services/Config/getCachedTools'); const { needsRefresh, getNewS3URL } = require('~/server/services/Files/S3/crud'); +const { needsRefreshAzure, getNewAzureURL } = require('~/server/services/Files/Azure/crud'); + +/** DI maps: dispatch avatar URL check and refresh by storage source */ +const avatarNeedsRefreshBySource = { + [FileSources.s3]: (url) => needsRefresh(url, 3600), + [FileSources.azure_blob]: (url) => needsRefreshAzure(url, 3600), +}; +const getNewAvatarUrlBySource = { + [FileSources.s3]: getNewS3URL, + [FileSources.azure_blob]: getNewAzureURL, +}; const { processDeleteRequest } = require('~/server/services/Files/process'); const { getAppConfig } = require('~/server/services/Config'); const { deleteToolCalls } = require('~/models/ToolCall'); @@ -56,20 +67,25 @@ const getUserController = async (req, res) => { delete userData.password; delete userData.totpSecret; delete userData.backupCodes; - if (appConfig.fileStrategy === FileSources.s3 && userData.avatar) { - const avatarNeedsRefresh = needsRefresh(userData.avatar, 3600); - if (!avatarNeedsRefresh) { + + // Signed-URL avatar refresh (S3 / Azure Blob) — dispatches by active file strategy + const strategy = appConfig.fileStrategy; + const checkNeedsRefresh = avatarNeedsRefreshBySource[strategy]; + const getNewAvatarUrl = getNewAvatarUrlBySource[strategy]; + if (checkNeedsRefresh && getNewAvatarUrl && userData.avatar) { + if (!checkNeedsRefresh(userData.avatar)) { return res.status(200).send(userData); } const originalAvatar = userData.avatar; try { - userData.avatar = await getNewS3URL(userData.avatar); + userData.avatar = await getNewAvatarUrl(userData.avatar); await updateUser(userData.id, { avatar: userData.avatar }); } catch (error) { userData.avatar = originalAvatar; - logger.error('Error getting new S3 URL for avatar:', error); + logger.error(`Error refreshing ${strategy} avatar URL:`, error); } } + res.status(200).send(userData); }; diff --git a/api/server/controllers/agents/v1.js b/api/server/controllers/agents/v1.js index a2c0d55186f8..0c73cd30ea67 100644 --- a/api/server/controllers/agents/v1.js +++ b/api/server/controllers/agents/v1.js @@ -47,6 +47,7 @@ const { refreshS3Url } = require('~/server/services/Files/S3/crud'); const { filterFile } = require('~/server/services/Files/process'); const { updateAction, getActions } = require('~/models/Action'); const { getCachedTools } = require('~/server/services/Config'); +const { refreshAzureUrl } = require('~/server/services/Files/Azure/crud'); const { getLogStores } = require('~/cache'); const systemTools = { @@ -58,6 +59,12 @@ const systemTools = { const MAX_SEARCH_LEN = 100; const escapeRegex = (str = '') => str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +/** DI map: dispatches avatar URL refresh by storage source */ +const urlRefreshersBySource = { + [FileSources.s3]: refreshS3Url, + [FileSources.azure_blob]: refreshAzureUrl, +}; + /** * Creates an Agent. * @route POST /Agents @@ -160,14 +167,13 @@ const getAgentHandler = async (req, res, expandProperties = false) => { agent.version = agent.versions ? agent.versions.length : 0; - if (agent.avatar && agent.avatar?.source === FileSources.s3) { + const refreshFn = agent.avatar && urlRefreshersBySource[agent.avatar.source]; + if (refreshFn) { try { - agent.avatar = { - ...agent.avatar, - filepath: await refreshS3Url(agent.avatar), - }; + const newPath = await refreshFn(agent.avatar); + agent.avatar = { ...agent.avatar, filepath: newPath }; } catch (e) { - logger.warn('[/Agents/:id] Failed to refresh S3 URL', e); + logger.warn('[/Agents/:id] Failed to refresh signed URL', e); } } @@ -545,6 +551,7 @@ const getListAgentsHandler = async (req, res) => { agents: fullList?.data ?? [], userId, refreshS3Url, + refreshAzureUrl, updateAgent, }); cachedRefresh = { urlCache }; @@ -553,7 +560,7 @@ const getListAgentsHandler = async (req, res) => { logger.error('[/Agents] Error refreshing avatars for full list: %o', err); } } else { - logger.debug('[/Agents] S3 avatar refresh already checked, skipping'); + logger.debug('[/Agents] Avatar refresh already checked, skipping'); } // Use the new ACL-aware function @@ -577,12 +584,7 @@ const getListAgentsHandler = async (req, res) => { if (agent?._id && publicSet.has(agent._id.toString())) { agent.isPublic = true; } - if ( - urlCache && - agent?.id && - agent?.avatar?.source === FileSources.s3 && - urlCache[agent.id] - ) { + if (urlCache && agent?.id && agent?.avatar && urlCache[agent.id]) { agent.avatar = { ...agent.avatar, filepath: urlCache[agent.id] }; } } catch (e) { diff --git a/api/server/routes/files/files.js b/api/server/routes/files/files.js index 5de2ddb3796d..f992ac7bf12e 100644 --- a/api/server/routes/files/files.js +++ b/api/server/routes/files/files.js @@ -3,7 +3,6 @@ const express = require('express'); const { EnvVar } = require('@librechat/agents'); const { logger } = require('@librechat/data-schemas'); const { - Time, isUUID, CacheKeys, FileSources, @@ -26,8 +25,10 @@ const { getOpenAIClient } = require('~/server/controllers/assistants/helpers'); const { checkPermission } = require('~/server/services/PermissionService'); const { loadAuthValues } = require('~/server/services/Tools/credentials'); const { refreshS3FileUrls } = require('~/server/services/Files/S3/crud'); +const { refreshAzureFileUrls } = require('~/server/services/Files/Azure/crud'); const { hasAccessToFilesViaAgent } = require('~/server/services/Files'); const { getFiles, batchUpdateFiles } = require('~/models'); +const { getFileURLRefreshCacheTime } = require('~/server/utils/getFileStrategy'); const { cleanFileName } = require('~/server/utils/files'); const { getAssistant } = require('~/models/Assistant'); const { getAgent } = require('~/models/Agent'); @@ -40,16 +41,24 @@ router.get('/', async (req, res) => { try { const appConfig = req.config; const files = await getFiles({ user: req.user.id }); - if (appConfig.fileStrategy === FileSources.s3) { + if (appConfig.fileStrategy === FileSources.s3 || appConfig.fileStrategy === FileSources.azure_blob) { try { - const cache = getLogStores(CacheKeys.S3_EXPIRY_INTERVAL); + const cacheKey = appConfig.fileStrategy === FileSources.s3 + ? CacheKeys.S3_EXPIRY_INTERVAL + : CacheKeys.AZURE_EXPIRY_INTERVAL; + const cache = getLogStores(cacheKey); const alreadyChecked = await cache.get(req.user.id); if (!alreadyChecked) { - await refreshS3FileUrls(files, batchUpdateFiles); - await cache.set(req.user.id, true, Time.THIRTY_MINUTES); + if (appConfig.fileStrategy === FileSources.s3) { + await refreshS3FileUrls(files, batchUpdateFiles); + } else { + await refreshAzureFileUrls(files, batchUpdateFiles); + } + const cacheTime = getFileURLRefreshCacheTime(appConfig.fileStrategy); + await cache.set(req.user.id, true, cacheTime); } } catch (error) { - logger.warn('[/files] Error refreshing S3 file URLs:', error); + logger.warn('[/files] Error refreshing file URLs for strategy:', appConfig.fileStrategy, error); } } res.status(200).send(files); diff --git a/api/server/services/Files/Azure/crud.js b/api/server/services/Files/Azure/crud.js index 8f681bd06c64..275c6a3bbade 100644 --- a/api/server/services/Files/Azure/crud.js +++ b/api/server/services/Files/Azure/crud.js @@ -4,10 +4,160 @@ const mime = require('mime'); const axios = require('axios'); const fetch = require('node-fetch'); const { logger } = require('@librechat/data-schemas'); +const { FileSources } = require('librechat-data-provider'); const { getAzureContainerClient, deleteRagFile } = require('@librechat/api'); const defaultBasePath = 'images'; const { AZURE_STORAGE_PUBLIC_ACCESS = 'true', AZURE_CONTAINER_NAME = 'files' } = process.env; +const { BlobSASPermissions, generateBlobSASQueryParameters, StorageSharedKeyCredential } = require('@azure/storage-blob'); + +let azureUrlExpirySeconds = 2 * 60; +let azureRefreshExpiryMs = null; + +let userDelegationKey = null; +let userDelegationKeyExpiry = null; +const DELEGATION_KEY_REFRESH_BUFFER_MS = 5 * 60 * 1000; // 5 minutes + +if (process.env.AZURE_URL_EXPIRY_SECONDS !== undefined) { + const parsed = parseInt(process.env.AZURE_URL_EXPIRY_SECONDS, 10); + if (!isNaN(parsed) && parsed > 0) { + azureUrlExpirySeconds = Math.min(parsed, 7 * 24 * 60 * 60); + } else { + logger.warn( + `[Azure] Invalid AZURE_URL_EXPIRY_SECONDS value: "${process.env.AZURE_URL_EXPIRY_SECONDS}". Using 2-minute expiry.`, + ); + } +} + +if (process.env.AZURE_REFRESH_EXPIRY_MS !== null && process.env.AZURE_REFRESH_EXPIRY_MS) { + const parsed = parseInt(process.env.AZURE_REFRESH_EXPIRY_MS, 10); + if (!isNaN(parsed) && parsed > 0) { + azureRefreshExpiryMs = parsed; + logger.info(`[Azure] Using custom refresh expiry time: ${azureRefreshExpiryMs}ms`); + } +} + +const isPublicAccess = () => AZURE_STORAGE_PUBLIC_ACCESS?.toLowerCase() === 'true'; + +/** + * Obtains and caches a User Delegation Key from Azure Blob Storage for use with + * Managed Identity (User Delegation SAS) URL signing. The key is cached for its + * full 7-day lifespan and proactively refreshed within a 5-minute buffer of expiry. + * + * @param {import('@azure/storage-blob').BlobServiceClient} blobServiceClient + * @returns {Promise} + */ +async function getUserDelegationKey(blobServiceClient) { + const now = new Date(); + + if (userDelegationKey && userDelegationKeyExpiry) { + const timeUntilExpiry = userDelegationKeyExpiry.getTime() - now.getTime(); + if (timeUntilExpiry > DELEGATION_KEY_REFRESH_BUFFER_MS) { + return userDelegationKey; + } + } + + const startsOn = now; + const expiresOn = new Date(now.getTime() + 7 * 24 * 60 * 60 * 1000); // 7 days + + userDelegationKey = await blobServiceClient.getUserDelegationKey(startsOn, expiresOn); + userDelegationKeyExpiry = expiresOn; + + logger.info('[Azure] User delegation key obtained, expires:', expiresOn.toISOString()); + return userDelegationKey; +} + +/** + * Generates a time-limited signed URL (SAS token) for an Azure Blob. + * Automatically selects the signing method based on configuration: + * - Account Key signing when `AZURE_STORAGE_CONNECTION_STRING` is set + * - User Delegation SAS via Managed Identity when only `AZURE_STORAGE_ACCOUNT_NAME` is set + * + * @param {Object} params + * @param {string} params.blobPath - The path of the blob within the container. + * @param {string} [params.containerName=AZURE_CONTAINER_NAME] - The Azure Blob Storage container name. + * @returns {Promise} A promise that resolves to the generated signed URL. + */ +async function getSignedAzureURL({ blobPath, containerName = AZURE_CONTAINER_NAME }) { + try { + const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING; + const accountName = process.env.AZURE_STORAGE_ACCOUNT_NAME; + + // BlobServiceClient is loaded dynamically to allow optional Azure SDK installation. + // BlobSASPermissions, generateBlobSASQueryParameters, and StorageSharedKeyCredential + // are imported at the top of the module and reused here. + const { BlobServiceClient } = await import('@azure/storage-blob'); + + const startsOn = new Date(); + const expiresOn = new Date(startsOn.getTime() + azureUrlExpirySeconds * 1000); + + if (connectionString) { + const blobServiceClient = BlobServiceClient.fromConnectionString(connectionString); + const containerClient = blobServiceClient.getContainerClient(containerName); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + + const match = connectionString.match(/AccountName=([^;]+)/i); + const keyMatch = connectionString.match(/AccountKey=([^;]+)/i); + + if (!match || !keyMatch) { + logger.warn('[getSignedAzureURL] Connection string missing AccountName or AccountKey, returning unsigned URL'); + return blockBlobClient.url; + } + + const sharedKeyCredential = new StorageSharedKeyCredential(match[1], keyMatch[1]); + + const sasToken = generateBlobSASQueryParameters( + { + containerName, + blobName: blobPath, + permissions: BlobSASPermissions.parse('r'), + startsOn, + expiresOn, + }, + sharedKeyCredential, + ).toString(); + + return `${blockBlobClient.url}?${sasToken}`; + } + + if (accountName) { + try { + const { DefaultAzureCredential } = await import('@azure/identity'); + const credential = new DefaultAzureCredential(); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + credential, + ); + + const delegationKey = await getUserDelegationKey(blobServiceClient); + const containerClient = blobServiceClient.getContainerClient(containerName); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + + const sasToken = generateBlobSASQueryParameters( + { + containerName, + blobName: blobPath, + permissions: BlobSASPermissions.parse('r'), + startsOn, + expiresOn, + }, + delegationKey, + accountName, + ).toString(); + + return `${blockBlobClient.url}?${sasToken}`; + } catch (credentialError) { + logger.error('[getSignedAzureURL] User Delegation signing failed. Ensure you are running on Azure with Managed Identity or use a connection string:', credentialError.message); + throw credentialError; + } + } + + throw new Error('Azure storage not configured: set AZURE_STORAGE_CONNECTION_STRING for local/AccountKey signing, or AZURE_STORAGE_ACCOUNT_NAME for Managed Identity'); + } catch (error) { + logger.error('[getSignedAzureURL] Error generating signed URL:', error); + throw error; + } +} /** * Uploads a buffer to Azure Blob Storage. @@ -31,12 +181,15 @@ async function saveBufferToAzure({ }) { try { const containerClient = await getAzureContainerClient(containerName); - const access = AZURE_STORAGE_PUBLIC_ACCESS?.toLowerCase() === 'true' ? 'blob' : undefined; - // Create the container if it doesn't exist. This is done per operation. + const access = isPublicAccess() ? 'blob' : undefined; await containerClient.createIfNotExists({ access }); const blobPath = `${basePath}/${userId}/${fileName}`; const blockBlobClient = containerClient.getBlockBlobClient(blobPath); await blockBlobClient.uploadData(buffer); + + if (!isPublicAccess()) { + return await getSignedAzureURL({ blobPath, containerName }); + } return blockBlobClient.url; } catch (error) { logger.error('[saveBufferToAzure] Error uploading buffer:', error); @@ -143,25 +296,20 @@ async function streamFileToAzure({ }) { try { const containerClient = await getAzureContainerClient(containerName); - const access = AZURE_STORAGE_PUBLIC_ACCESS?.toLowerCase() === 'true' ? 'blob' : undefined; - - // Create the container if it doesn't exist + const access = isPublicAccess() ? 'blob' : undefined; await containerClient.createIfNotExists({ access }); const blobPath = `${basePath}/${userId}/${fileName}`; const blockBlobClient = containerClient.getBlockBlobClient(blobPath); - // Get file size for proper content length const stats = await fs.promises.stat(filePath); - - // Create read stream from the file const fileStream = fs.createReadStream(filePath); const blobContentType = mime.getType(fileName); await blockBlobClient.uploadStream( fileStream, - undefined, // Use default concurrency (5) - undefined, // Use default buffer size (8MB) + undefined, + undefined, { blobHTTPHeaders: { blobContentType, @@ -174,6 +322,9 @@ async function streamFileToAzure({ }, ); + if (!isPublicAccess()) { + return await getSignedAzureURL({ blobPath, containerName }); + } return blockBlobClient.url; } catch (error) { logger.error('[streamFileToAzure] Error streaming file:', error); @@ -245,6 +396,194 @@ async function getAzureFileStream(_req, fileURL) { } } +/** + * Determines whether the provided Azure Blob Storage URL needs to be refreshed. + * A plain URL without a SAS token is treated as needing a refresh when private + * access is enabled, allowing seamless transitions from public to private access. + * + * @param {string} signedUrl - The Azure Blob Storage URL, which may include a SAS token. + * @param {number} bufferSeconds - The number of seconds before expiration at which the URL + * should be considered in need of refresh. + * @returns {boolean} `true` if the URL should be refreshed, `false` if it is still valid. + */ +function needsRefreshAzure(signedUrl, bufferSeconds) { + try { + const url = new URL(signedUrl); + const hasSasToken = url.searchParams.has('se'); + + // Private access required but URL is plain → needs signing + if (!isPublicAccess() && !hasSasToken) { + return true; + } + + // Public access and no SAS token → no refresh needed + if (!hasSasToken) { + return false; + } + + // Check expiration for signed URLs + const expiresParam = url.searchParams.get('se'); + if (!expiresParam) { + return true; + } + + const expiresAtDate = new Date(expiresParam); + const now = new Date(); + + if (azureRefreshExpiryMs !== null) { + const stParam = url.searchParams.get('st'); + if (stParam) { + const urlCreationTime = new Date(stParam).getTime(); + const urlAge = now.getTime() - urlCreationTime; + return urlAge >= azureRefreshExpiryMs; + } + } + + const bufferTime = new Date(now.getTime() + bufferSeconds * 1000); + return expiresAtDate <= bufferTime; + } catch (error) { + logger.error('[needsRefreshAzure] Error checking URL expiration:', error); + return true; + } +} + +/** + * Extracts the blob path from an Azure Blob Storage URL. + * + * For a URL of the form `https://.blob.core.windows.net//`, + * this function returns the `` portion, or `null` if it cannot be determined. + * + * @param {string} fileUrl - The full Azure Blob Storage URL of the file. + * @returns {string | null} The blob path within the container, or `null` on failure. + */ +function extractBlobPathFromAzureUrl(fileUrl) { + try { + const url = new URL(fileUrl); + const pathname = url.pathname; + const parts = pathname.split('/').filter(Boolean); + if (parts.length < 2) { + return null; + } + return parts.slice(1).join('/'); + } catch (error) { + logger.error('[extractBlobPathFromAzureUrl] Error extracting blob path:', error); + return null; + } +} + +/** + * Generates a new Azure Blob URL for the given file URL. + * Returns a signed (SAS) URL when private access is enabled, or a plain blob URL + * when public access is configured. + * + * @param {string} currentURL - The existing Azure Blob file URL that may need to be refreshed or signed. + * @returns {Promise} A promise that resolves to the new URL, + * or `undefined` if the blob path cannot be extracted. + */ +async function getNewAzureURL(currentURL) { + try { + const blobPath = extractBlobPathFromAzureUrl(currentURL); + if (!blobPath) { + return; + } + + if (!isPublicAccess()) { + return await getSignedAzureURL({ blobPath }); + } + + // Return plain URL for public access + const containerClient = await getAzureContainerClient(); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + return blockBlobClient.url; + } catch (error) { + logger.error('[getNewAzureURL] Error getting new Azure URL:', error); + } +} + +/** + * Refreshes Azure Blob file URLs that are close to expiring and persists the updated URLs. + * + * @param {MongoFile[]} files - The list of file records whose Azure URLs may need refreshing. + * @param {(updates: Partial[]) => Promise} batchUpdateFiles - Function used to + * update the file records in bulk with their new URLs. + * @param {number} [bufferSeconds=3600] - The minimum number of seconds before expiry at which a + * URL should be refreshed. + * @returns {Promise} A promise that resolves to the array of files with refreshed URLs. + */ +async function refreshAzureFileUrls(files, batchUpdateFiles, bufferSeconds = 3600) { + if (!files || !Array.isArray(files) || files.length === 0) { + return files; + } + + const filesToUpdate = []; + + for (let i = 0; i < files.length; i++) { + const file = files[i]; + if (!file?.file_id || file.source !== FileSources.azure_blob || !file.filepath) { + continue; + } + if (!needsRefreshAzure(file.filepath, bufferSeconds)) { + continue; + } + try { + const newURL = await getNewAzureURL(file.filepath); + if (!newURL) { + continue; + } + filesToUpdate.push({ + file_id: file.file_id, + filepath: newURL, + }); + files[i].filepath = newURL; + } catch (error) { + logger.error(`[refreshAzureFileUrls] Error refreshing Azure URL for file ${file.file_id}:`, error); + } + } + + if (filesToUpdate.length > 0) { + await batchUpdateFiles(filesToUpdate); + } + + return files; +} + +/** + * Refreshes an Azure Blob storage file URL if it is close to expiration. + * + * When the file originates from Azure Blob storage and the existing URL is determined to be + * near expiry (based on {@link needsRefreshAzure} and the provided `bufferSeconds`), this + * function generates and returns a new URL. If the URL does not need to be refreshed or a + * new URL cannot be obtained, the original URL is returned instead. + * + * @param {Object} fileObj - The file metadata object. + * @param {string} fileObj.filepath - The current Azure Blob file URL that may need refreshing. + * @param {string} fileObj.source - The source type; must be {@link FileSources.azure_blob} to trigger a refresh. + * @param {number} [bufferSeconds=3600] - The number of seconds before URL expiry within which a refresh should be attempted. + * @returns {Promise} A promise that resolves to the (possibly refreshed) Azure Blob file URL. + */ +async function refreshAzureUrl(fileObj, bufferSeconds = 3600) { + if (!fileObj || fileObj.source !== FileSources.azure_blob || !fileObj.filepath) { + return fileObj?.filepath || ''; + } + + if (!needsRefreshAzure(fileObj.filepath, bufferSeconds)) { + return fileObj.filepath; + } + + try { + const newUrl = await getNewAzureURL(fileObj.filepath); + if (!newUrl) { + logger.warn(`[refreshAzureUrl] Unable to refresh Azure URL: ${fileObj.filepath}`); + return fileObj.filepath; + } + logger.debug(`[refreshAzureUrl] Refreshed Azure URL`); + return newUrl; + } catch (error) { + logger.error(`[refreshAzureUrl] Error refreshing Azure URL: ${error.message}`); + return fileObj.filepath; + } +} + module.exports = { saveBufferToAzure, saveURLToAzure, @@ -252,4 +591,10 @@ module.exports = { deleteFileFromAzure, uploadFileToAzure, getAzureFileStream, + getSignedAzureURL, + needsRefreshAzure, + refreshAzureFileUrls, + refreshAzureUrl, + getNewAzureURL, + extractBlobPathFromAzureUrl }; diff --git a/api/server/services/Files/Azure/images.js b/api/server/services/Files/Azure/images.js index 4d857d0d1db6..e69476e3a058 100644 --- a/api/server/services/Files/Azure/images.js +++ b/api/server/services/Files/Azure/images.js @@ -122,15 +122,14 @@ async function processAzureAvatar({ basePath, containerName, }); - const isManual = manual === 'true'; - const url = `${downloadURL}?manual=${isManual}`; // Only update user record if this is a user avatar (manual === 'true') + const isManual = manual === 'true'; if (isManual && !agentId) { - await updateUser(userId, { avatar: url }); + await updateUser(userId, { avatar: downloadURL }); } - return url; + return downloadURL; } catch (error) { logger.error('[processAzureAvatar] Error uploading profile picture to Azure:', error); throw error; diff --git a/api/server/utils/getFileStrategy.js b/api/server/utils/getFileStrategy.js index 2e3dfdd79eb4..67d5e2862959 100644 --- a/api/server/utils/getFileStrategy.js +++ b/api/server/utils/getFileStrategy.js @@ -1,4 +1,4 @@ -const { FileSources, FileContext } = require('librechat-data-provider'); +const { Time, FileSources, FileContext } = require('librechat-data-provider'); /** * Determines the appropriate file storage strategy based on file type and configuration. @@ -53,4 +53,26 @@ function getFileStrategy(appConfig, { isAvatar = false, isImage = false, context return selectedStrategy || FileSources.local; // Final fallback to FileSources.local } -module.exports = { getFileStrategy }; +/** + * Calculates the cache duration for signed file URL refresh checks. + * Uses half the URL expiry time to ensure URLs are refreshed before they expire, + * with a minimum of one minute. + * + * @param {string} fileStrategy - The active file storage strategy (e.g. FileSources.s3, FileSources.azure_blob) + * @returns {number} Cache duration in milliseconds + */ +function getFileURLRefreshCacheTime(fileStrategy) { + if (fileStrategy === FileSources.s3) { + const expirySeconds = parseInt(process.env.S3_URL_EXPIRY_SECONDS, 10) || 120; + // Check for refresh at half the URL expiry time: (expirySeconds * 1000ms) / 2 => expirySeconds * 500 + return Math.max(expirySeconds * 500, Time.ONE_MINUTE); + } + if (fileStrategy === FileSources.azure_blob) { + const expirySeconds = parseInt(process.env.AZURE_URL_EXPIRY_SECONDS, 10) || 120; + // Check for refresh at half the URL expiry time: (expirySeconds * 1000ms) / 2 => expirySeconds * 500 + return Math.max(expirySeconds * 500, Time.ONE_MINUTE); + } + return Time.THIRTY_MINUTES; +} + +module.exports = { getFileStrategy, getFileURLRefreshCacheTime }; diff --git a/api/test/server/services/Files/Azure/crud.test.js b/api/test/server/services/Files/Azure/crud.test.js new file mode 100644 index 000000000000..f11cdcbe943c --- /dev/null +++ b/api/test/server/services/Files/Azure/crud.test.js @@ -0,0 +1,109 @@ +const { + needsRefreshAzure, + extractBlobPathFromAzureUrl, + getNewAzureURL, +} = require('../../../../../server/services/Files/Azure/crud'); + +jest.mock('@librechat/api', () => ({ + getAzureContainerClient: jest.fn(), + deleteRagFile: jest.fn(), +})); + +const { getAzureContainerClient } = require('@librechat/api'); + +describe('Azure crud.js - URL refresh tests', () => { + const originalEnv = process.env; + + beforeEach(() => { + jest.clearAllMocks(); + process.env = { ...originalEnv }; + }); + + afterEach(() => { + process.env = originalEnv; + }); + + describe('needsRefreshAzure', () => { + it('should return false for URLs without SAS token when public access enabled', () => { + process.env.AZURE_STORAGE_PUBLIC_ACCESS = 'true'; + const url = 'https://test.blob.core.windows.net/files/images/user123/test.pdf'; + expect(needsRefreshAzure(url, 3600)).toBe(false); + }); + + it('should return true for expired URLs', () => { + const pastDate = new Date(Date.now() - 3600000).toISOString(); + const url = `https://test.blob.core.windows.net/files/test.pdf?se=${encodeURIComponent(pastDate)}&sp=r&sig=test`; + expect(needsRefreshAzure(url, 3600)).toBe(true); + }); + + it('should return false for URLs that are not close to expiring', () => { + const futureDate = new Date(Date.now() + 7200000).toISOString(); + const url = `https://test.blob.core.windows.net/files/test.pdf?se=${encodeURIComponent(futureDate)}&sp=r&sig=test`; + expect(needsRefreshAzure(url, 3600)).toBe(false); + }); + + it('should return true for URLs within buffer time', () => { + const nearFutureDate = new Date(Date.now() + 1800000).toISOString(); + const url = `https://test.blob.core.windows.net/files/test.pdf?se=${encodeURIComponent(nearFutureDate)}&sp=r&sig=test`; + expect(needsRefreshAzure(url, 3600)).toBe(true); + }); + + it('should return true for invalid URLs', () => { + expect(needsRefreshAzure('not-a-valid-url', 3600)).toBe(true); + }); + }); + + describe('extractBlobPathFromAzureUrl', () => { + it('should extract blob path from Azure URL', () => { + const url = 'https://test.blob.core.windows.net/files/images/user123/test.pdf'; + expect(extractBlobPathFromAzureUrl(url)).toBe('images/user123/test.pdf'); + }); + + it('should extract blob path from URL with SAS token', () => { + const url = 'https://test.blob.core.windows.net/files/images/user123/test.pdf?sv=2021&sig=abc'; + expect(extractBlobPathFromAzureUrl(url)).toBe('images/user123/test.pdf'); + }); + + it('should return null for URL with only container', () => { + const url = 'https://test.blob.core.windows.net/files'; + expect(extractBlobPathFromAzureUrl(url)).toBe(null); + }); + + it('should return null for invalid URL', () => { + expect(extractBlobPathFromAzureUrl('not-a-url')).toBe(null); + }); + }); + + describe('needsRefreshAzure - public to private transition', () => { + it('should return true for plain URL when private access is required', () => { + process.env.AZURE_STORAGE_PUBLIC_ACCESS = 'false'; + const plainUrl = 'https://test.blob.core.windows.net/files/images/user123/test.pdf'; + expect(needsRefreshAzure(plainUrl, 3600)).toBe(true); + }); + + it('should return false for plain URL when public access is enabled', () => { + process.env.AZURE_STORAGE_PUBLIC_ACCESS = 'true'; + const plainUrl = 'https://test.blob.core.windows.net/files/images/user123/test.pdf'; + expect(needsRefreshAzure(plainUrl, 3600)).toBe(false); + }); + }); + + describe('getNewAzureURL', () => { + it('should return undefined for a URL with no extractable blob path', async () => { + const invalidUrl = 'https://test.blob.core.windows.net/files'; + const result = await getNewAzureURL(invalidUrl); + expect(result).toBeUndefined(); + }); + + it('should return a plain blob URL when public access is enabled', async () => { + process.env.AZURE_STORAGE_PUBLIC_ACCESS = 'true'; + const mockBlockBlobClient = { url: 'https://test.blob.core.windows.net/files/images/user123/test.pdf' }; + const mockContainerClient = { getBlockBlobClient: jest.fn().mockReturnValue(mockBlockBlobClient) }; + getAzureContainerClient.mockResolvedValue(mockContainerClient); + + const url = 'https://test.blob.core.windows.net/files/images/user123/test.pdf'; + const result = await getNewAzureURL(url); + expect(result).toBe(mockBlockBlobClient.url); + }); + }); +}); \ No newline at end of file diff --git a/packages/api/src/agents/avatars.ts b/packages/api/src/agents/avatars.ts index 25adfdc7175e..0a1912556beb 100644 --- a/packages/api/src/agents/avatars.ts +++ b/packages/api/src/agents/avatars.ts @@ -9,6 +9,9 @@ export { MAX_AVATAR_REFRESH_AGENTS, AVATAR_REFRESH_BATCH_SIZE }; export type RefreshS3UrlFn = (avatar: AgentAvatar) => Promise; +/** Generic signed URL refresh function, agnostic to storage provider */ +export type RefreshUrlFn = (avatar: AgentAvatar) => Promise; + export type UpdateAgentFn = ( searchParams: { id: string }, updateData: { avatar: AgentAvatar }, @@ -19,6 +22,8 @@ export type RefreshListAvatarsParams = { agents: Agent[]; userId: string; refreshS3Url: RefreshS3UrlFn; + /** Optional Azure Blob URL refresher; when provided, Azure-backed avatars are also refreshed */ + refreshAzureUrl?: RefreshUrlFn; updateAgent: UpdateAgentFn; }; @@ -48,6 +53,7 @@ export const refreshListAvatars = async ({ agents, userId, refreshS3Url, + refreshAzureUrl, updateAgent, }: RefreshListAvatarsParams): Promise => { const stats: RefreshStats = { @@ -71,23 +77,29 @@ export const refreshListAvatars = async ({ await Promise.all( batch.map(async (agent) => { - if (agent?.avatar?.source !== FileSources.s3 || !agent?.avatar?.filepath) { + const source = agent?.avatar?.source; + const isS3 = source === FileSources.s3; + const isAzure = source === FileSources.azure_blob && !!refreshAzureUrl; + + if ((!isS3 && !isAzure) || !agent?.avatar?.filepath) { stats.not_s3++; return; } if (!agent?.id) { logger.debug( - '[refreshListAvatars] Skipping S3 avatar refresh for agent: %s, ID is not set', + '[refreshListAvatars] Skipping avatar refresh for agent: %s, ID is not set', agent._id, ); stats.no_id++; return; } + const refreshFn = isS3 ? refreshS3Url : refreshAzureUrl!; + try { - logger.debug('[refreshListAvatars] Refreshing S3 avatar for agent: %s', agent._id); - const newPath = await refreshS3Url(agent.avatar); + logger.debug('[refreshListAvatars] Refreshing avatar for agent: %s', agent._id); + const newPath = await refreshFn(agent.avatar); if (!newPath || newPath === agent.avatar.filepath) { stats.no_change++; @@ -108,7 +120,7 @@ export const refreshListAvatars = async ({ stats.persist_error++; } } catch (err) { - logger.error('[refreshListAvatars] S3 avatar refresh error: %o', err); + logger.error('[refreshListAvatars] Avatar refresh error: %o', err); stats.s3_error++; } }), diff --git a/packages/data-provider/src/config.ts b/packages/data-provider/src/config.ts index 64fc99b0ebb9..c858df6e58cb 100644 --- a/packages/data-provider/src/config.ts +++ b/packages/data-provider/src/config.ts @@ -1443,6 +1443,10 @@ export enum CacheKeys { * Key for s3 check intervals per user */ S3_EXPIRY_INTERVAL = 'S3_EXPIRY_INTERVAL', + /** + * Key for Azure Blob check intervals per user + */ + AZURE_EXPIRY_INTERVAL = 'AZURE_EXPIRY_INTERVAL', /** * key for open id exchanged tokens */