// /utils/s3Client.js const fs = require('fs'); const { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, ListObjectsV2Command, HeadBucketCommand, CreateBucketCommand } = require('@aws-sdk/client-s3'); const { getSignedUrl } = require('@aws-sdk/s3-request-presigner'); const config = require('../config/config'); const logger = require('./logger'); const AppError = require('./AppError'); let s3ClientInstance = null; const BUCKETS = { temp: () => config.S3_TEMP_BUCKET, certificates: () => config.S3_CERTIFICATES_BUCKET, documents: () => config.S3_DOCUMENTS_BUCKET, storage: () => config.S3_STORAGE_BUCKET }; const resolveBucket = (bucketNameOrKind) => { if (!bucketNameOrKind) return config.S3_CERTIFICATES_BUCKET; if (BUCKETS[bucketNameOrKind]) return BUCKETS[bucketNameOrKind](); return bucketNameOrKind; }; const isBucketPublic = (bucketName) => { const name = resolveBucket(bucketName); if (name === config.S3_CERTIFICATES_BUCKET) return config.S3_CERTIFICATES_PUBLIC; if (name === config.S3_DOCUMENTS_BUCKET) return config.S3_DOCUMENTS_PUBLIC; return false; }; const buildPublicUrl = (bucketName, fileKey) => { if (!isBucketPublic(bucketName) || !fileKey) return null; const base = (config.S3_PUBLIC_BASE_URL || config.S3_ENDPOINT || '').replace(/\/$/, ''); if (!base) return null; return `${base}/${resolveBucket(bucketName)}/${fileKey}`; }; const getS3Client = () => { if (!s3ClientInstance) { const endpoint = String(config.S3_ENDPOINT || '').replace(/\/$/, ''); s3ClientInstance = new S3Client({ endpoint, region: config.S3_REGION || 'us-east-1', credentials: { accessKeyId: config.S3_ACCESS_KEY, secretAccessKey: config.S3_SECRET_KEY }, forcePathStyle: config.S3_FORCE_PATH_STYLE !== false, // SeaweedFS / MinIO: AWS SDK v3 default flexible checksums break PutObject // (XML parse / unexpected content). Only checksum when the API requires it. // See: https://github.com/seaweedfs/seaweedfs/wiki/nodejs-with-Seaweed-S3 requestChecksumCalculation: 'WHEN_REQUIRED', responseChecksumValidation: 'WHEN_REQUIRED', useDualstackEndpoint: false }); logger.info(`[S3 Storage] Client ready → ${endpoint} (pathStyle=${config.S3_FORCE_PATH_STYLE !== false})`); } return s3ClientInstance; }; const previewErrorBody = (body) => { if (!body) return ''; if (typeof body === 'string') return body.slice(0, 300); if (Buffer.isBuffer(body)) return body.toString('utf8').slice(0, 300); try { return JSON.stringify(body).slice(0, 300); } catch { return Object.prototype.toString.call(body); } }; const logS3Error = (label, error) => { const status = error?.$metadata?.httpStatusCode; const raw = error?.$response; let bodyPreview = ''; try { bodyPreview = previewErrorBody(raw?.body || raw?.reason || ''); } catch { bodyPreview = ''; } logger.error( `[S3 Storage ERROR] ${label}: ${error.message}` + (status ? ` (HTTP ${status})` : '') + (error?.name ? ` name=${error.name}` : '') + (error?.Code ? ` code=${error.Code}` : '') + (bodyPreview ? ` body=${bodyPreview.replace(/\s+/g, ' ')}` : '') ); if (status === 307 || /Temporary Redirect/i.test(String(error.message) + bodyPreview)) { logger.error( '[S3 Storage ERROR] Hint: HTTP 307 means S3_ENDPOINT is redirecting (often http→https). ' + `Set S3_ENDPOINT=https://… (current: ${config.S3_ENDPOINT}). AWS SDK does not follow PUT redirects.` ); } }; const streamToBuffer = async (body) => { if (!body) return Buffer.alloc(0); if (Buffer.isBuffer(body)) return body; if (typeof body.transformToByteArray === 'function') { return Buffer.from(await body.transformToByteArray()); } const chunks = []; for await (const chunk of body) { chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); } return Buffer.concat(chunks); }; const ensuredBuckets = new Set(); const ensureBucket = async (bucketNameOrKind) => { const bucket = resolveBucket(bucketNameOrKind); if (ensuredBuckets.has(bucket)) return bucket; const client = getS3Client(); try { await client.send(new HeadBucketCommand({ Bucket: bucket })); ensuredBuckets.add(bucket); return bucket; } catch (headErr) { logS3Error(`HeadBucket ${bucket} (will try create)`, headErr); } try { await client.send(new CreateBucketCommand({ Bucket: bucket })); logger.info(`[S3 Storage] Created bucket ${bucket}`); } catch (createErr) { const code = createErr?.name || createErr?.Code || ''; if (code !== 'BucketAlreadyOwnedByYou' && code !== 'BucketAlreadyExists') { logS3Error(`Failed to create bucket ${bucket}`, createErr); throw createErr; } } ensuredBuckets.add(bucket); return bucket; }; const ensureAppBuckets = async () => { const names = [ config.S3_TEMP_BUCKET, config.S3_CERTIFICATES_BUCKET, config.S3_DOCUMENTS_BUCKET ].filter(Boolean); for (const name of [...new Set(names)]) { try { await ensureBucket(name); logger.info(`[S3 Storage] Bucket ready: ${name}`); } catch (err) { logger.error(`[S3 Storage] Could not ensure bucket ${name}: ${err.message}`); } } }; const toPutBody = (fileBufferOrPath) => { if (typeof fileBufferOrPath === 'string') { return fs.createReadStream(fileBufferOrPath); } return fileBufferOrPath; }; const uploadToTempBucket = async (fileBufferOrPath, filename, contentType = 'application/octet-stream') => { try { await ensureBucket(config.S3_TEMP_BUCKET); const client = getS3Client(); const command = new PutObjectCommand({ Bucket: config.S3_TEMP_BUCKET, Key: filename, Body: toPutBody(fileBufferOrPath), ContentType: contentType }); await client.send(command); logger.info(`[S3 Storage] Uploaded temp file: ${filename} to bucket ${config.S3_TEMP_BUCKET}`); return { tempFileName: filename, bucket: config.S3_TEMP_BUCKET }; } catch (error) { logS3Error(`Temp upload failed for ${filename}`, error); // Mock fallback for test environment when S3 is unavailable if (config.NODE_ENV === 'test' || error.code === 'ECONNREFUSED') { logger.warn(`[S3 Storage MOCK] Simulated temp upload for ${filename}`); return { tempFileName: filename, bucket: config.S3_TEMP_BUCKET }; } throw error; } }; const getObjectBuffer = async (filename, bucketName = config.S3_CERTIFICATES_BUCKET) => { if (!filename) { throw new AppError('VALIDATION_FAILED', null, 'Filename is required'); } const bucket = resolveBucket(bucketName); try { const client = getS3Client(); const result = await client.send(new GetObjectCommand({ Bucket: bucket, Key: filename })); const buffer = await streamToBuffer(result.Body); return { buffer, contentType: result.ContentType || 'application/octet-stream', bucket }; } catch (error) { logS3Error(`GetObject failed for ${bucket}/${filename}`, error); throw new AppError('FILE_NOT_FOUND'); } }; const rewriteTempObject = async (client, tempFilename, targetBucket, targetKey) => { const source = await client.send(new GetObjectCommand({ Bucket: config.S3_TEMP_BUCKET, Key: tempFilename })); const body = await streamToBuffer(source.Body); await client.send(new PutObjectCommand({ Bucket: targetBucket, Key: targetKey, Body: body, ContentType: source.ContentType || 'application/octet-stream' })); }; /** * Move a temp object into a target bucket, then remove the temp object. * SeaweedFS CopyObject returns InternalError 500 across buckets, so this * rewrites via GetObject + PutObject (PutObject already works for temp uploads). */ const commitTempFile = async (tempFilename, destinationKey = null, targetBucketKind = 'certificates') => { const targetKey = destinationKey || tempFilename; const targetBucket = resolveBucket(targetBucketKind); try { await ensureBucket(targetBucket); const client = getS3Client(); await rewriteTempObject(client, tempFilename, targetBucket, targetKey); try { await client.send(new DeleteObjectCommand({ Bucket: config.S3_TEMP_BUCKET, Key: tempFilename })); } catch (deleteError) { logger.warn(`[S3 Storage] Committed ${targetKey} but failed to delete temp ${tempFilename}: ${deleteError.message}`); } const fileUrl = buildPublicUrl(targetBucket, targetKey); logger.info(`[S3 Storage] Committed ${tempFilename} → ${targetBucket}/${targetKey}`); return { fileKey: targetKey, bucket: targetBucket, fileUrl }; } catch (error) { logS3Error(`Failed to commit temp file ${tempFilename}`, error); if (config.NODE_ENV === 'test' || error.code === 'ECONNREFUSED') { logger.warn(`[S3 Storage MOCK] Simulated file commit for ${targetKey}`); return { fileKey: targetKey, bucket: targetBucket, fileUrl: buildPublicUrl(targetBucket, targetKey) }; } throw new AppError('FILE_COMMIT_FAILED'); } }; const generatePresignedUrl = async ( filename, bucketName = config.S3_CERTIFICATES_BUCKET, expiresIn = config.SIGNED_URL_EXPIRES_IN ) => { const bucket = resolveBucket(bucketName); try { const client = getS3Client(); const command = new GetObjectCommand({ Bucket: bucket, Key: filename }); return await getSignedUrl(client, command, { expiresIn }); } catch (error) { logger.error(`[S3 Storage ERROR] Failed to generate presigned URL for ${filename}: ${error.message}`); if (config.NODE_ENV === 'test') { return `${config.S3_ENDPOINT}/${bucket}/${filename}?token=mock_presigned_${Date.now()}`; } throw error; } }; const deleteFromBucket = async (filename, bucketName = config.S3_CERTIFICATES_BUCKET) => { const bucket = resolveBucket(bucketName); try { const client = getS3Client(); const command = new DeleteObjectCommand({ Bucket: bucket, Key: filename }); await client.send(command); logger.info(`[S3 Storage] Deleted file ${filename} from bucket ${bucket}`); return true; } catch (error) { logger.error(`[S3 Storage ERROR] Failed to delete file ${filename}: ${error.message}`); return false; } }; const cleanupTempBucket = async (olderThanMinutes = 10) => { try { const client = getS3Client(); const listCommand = new ListObjectsV2Command({ Bucket: config.S3_TEMP_BUCKET }); const listResult = await client.send(listCommand); if (!listResult.Contents || listResult.Contents.length === 0) { logger.info('[S3 Temp Cleanup] Temp bucket is empty. Nothing to clean.'); return 0; } const cutoffTime = new Date(Date.now() - olderThanMinutes * 60 * 1000); let deletedCount = 0; for (const object of listResult.Contents) { if (object.LastModified && new Date(object.LastModified) < cutoffTime) { await deleteFromBucket(object.Key, config.S3_TEMP_BUCKET); deletedCount++; logger.info(`[S3 Temp Cleanup] Deleted expired temp file: ${object.Key}`); } } logger.info(`[S3 Temp Cleanup] Removed ${deletedCount} expired temp files.`); return deletedCount; } catch (error) { logger.error(`[S3 Temp Cleanup ERROR] Cleanup job failed: ${error.message}`); return 0; } }; module.exports = { getS3Client, uploadToTempBucket, commitTempFile, generatePresignedUrl, getObjectBuffer, deleteFromBucket, cleanupTempBucket, resolveBucket, isBucketPublic, buildPublicUrl, ensureBucket, ensureAppBuckets, BUCKETS };