http://s3 hosts redirect to https; AWS SDK cannot follow PUT redirects, which caused XML parse errors.
239 lines
7.9 KiB
JavaScript
239 lines
7.9 KiB
JavaScript
// /utils/s3Client.js
|
|
|
|
const {
|
|
S3Client,
|
|
PutObjectCommand,
|
|
GetObjectCommand,
|
|
CopyObjectCommand,
|
|
DeleteObjectCommand,
|
|
ListObjectsV2Command
|
|
} = require('@aws-sdk/client-s3');
|
|
const { getSignedUrl } = require('@aws-sdk/s3-request-presigner');
|
|
const config = require('../config/config');
|
|
const logger = require('./logger');
|
|
|
|
let s3ClientInstance = null;
|
|
|
|
const BUCKETS = {
|
|
temp: () => config.S3_TEMP_BUCKET,
|
|
certificates: () => config.S3_CERTIFICATES_BUCKET,
|
|
documents: () => config.S3_DOCUMENTS_BUCKET,
|
|
storage: () => config.S3_STORAGE_BUCKET
|
|
};
|
|
|
|
const resolveBucket = (bucketNameOrKind) => {
|
|
if (!bucketNameOrKind) return config.S3_CERTIFICATES_BUCKET;
|
|
if (BUCKETS[bucketNameOrKind]) return BUCKETS[bucketNameOrKind]();
|
|
return bucketNameOrKind;
|
|
};
|
|
|
|
const isBucketPublic = (bucketName) => {
|
|
const name = resolveBucket(bucketName);
|
|
if (name === config.S3_CERTIFICATES_BUCKET) return config.S3_CERTIFICATES_PUBLIC;
|
|
if (name === config.S3_DOCUMENTS_BUCKET) return config.S3_DOCUMENTS_PUBLIC;
|
|
return false;
|
|
};
|
|
|
|
const buildPublicUrl = (bucketName, fileKey) => {
|
|
if (!isBucketPublic(bucketName) || !fileKey) return null;
|
|
const base = (config.S3_PUBLIC_BASE_URL || config.S3_ENDPOINT || '').replace(/\/$/, '');
|
|
if (!base) return null;
|
|
return `${base}/${resolveBucket(bucketName)}/${fileKey}`;
|
|
};
|
|
|
|
const getS3Client = () => {
|
|
if (!s3ClientInstance) {
|
|
const endpoint = String(config.S3_ENDPOINT || '').replace(/\/$/, '');
|
|
s3ClientInstance = new S3Client({
|
|
endpoint,
|
|
region: config.S3_REGION || 'us-east-1',
|
|
credentials: {
|
|
accessKeyId: config.S3_ACCESS_KEY,
|
|
secretAccessKey: config.S3_SECRET_KEY
|
|
},
|
|
forcePathStyle: config.S3_FORCE_PATH_STYLE !== false,
|
|
// SeaweedFS / MinIO: AWS SDK v3 default flexible checksums break PutObject
|
|
// (XML parse / unexpected content). Only checksum when the API requires it.
|
|
// See: https://github.com/seaweedfs/seaweedfs/wiki/nodejs-with-Seaweed-S3
|
|
requestChecksumCalculation: 'WHEN_REQUIRED',
|
|
responseChecksumValidation: 'WHEN_REQUIRED',
|
|
useDualstackEndpoint: false
|
|
});
|
|
logger.info(`[S3 Storage] Client ready → ${endpoint} (pathStyle=${config.S3_FORCE_PATH_STYLE !== false})`);
|
|
}
|
|
return s3ClientInstance;
|
|
};
|
|
|
|
const logS3Error = (label, error) => {
|
|
const status = error?.$metadata?.httpStatusCode;
|
|
const raw = error?.$response;
|
|
let bodyPreview = '';
|
|
try {
|
|
const body = raw?.body || raw?.reason || '';
|
|
if (typeof body === 'string') bodyPreview = body.slice(0, 300);
|
|
else if (body && typeof body.toString === 'function') bodyPreview = String(body).slice(0, 300);
|
|
} catch {
|
|
bodyPreview = '';
|
|
}
|
|
logger.error(
|
|
`[S3 Storage ERROR] ${label}: ${error.message}`
|
|
+ (status ? ` (HTTP ${status})` : '')
|
|
+ (bodyPreview ? ` body=${bodyPreview.replace(/\s+/g, ' ')}` : '')
|
|
);
|
|
if (status === 307 || /Temporary Redirect/i.test(String(error.message) + bodyPreview)) {
|
|
logger.error(
|
|
'[S3 Storage ERROR] Hint: HTTP 307 means S3_ENDPOINT is redirecting (often http→https). '
|
|
+ `Set S3_ENDPOINT=https://… (current: ${config.S3_ENDPOINT}). AWS SDK does not follow PUT redirects.`
|
|
);
|
|
}
|
|
};
|
|
|
|
const uploadToTempBucket = async (fileBuffer, filename, contentType = 'application/octet-stream') => {
|
|
try {
|
|
const client = getS3Client();
|
|
const command = new PutObjectCommand({
|
|
Bucket: config.S3_TEMP_BUCKET,
|
|
Key: filename,
|
|
Body: fileBuffer,
|
|
ContentType: contentType
|
|
});
|
|
|
|
await client.send(command);
|
|
logger.info(`[S3 Storage] Uploaded temp file: ${filename} to bucket ${config.S3_TEMP_BUCKET}`);
|
|
return { tempFileName: filename, bucket: config.S3_TEMP_BUCKET };
|
|
} catch (error) {
|
|
logS3Error(`Temp upload failed for ${filename}`, error);
|
|
// Mock fallback for test environment when S3 is unavailable
|
|
if (config.NODE_ENV === 'test' || error.code === 'ECONNREFUSED') {
|
|
logger.warn(`[S3 Storage MOCK] Simulated temp upload for ${filename}`);
|
|
return { tempFileName: filename, bucket: config.S3_TEMP_BUCKET };
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Copy a temp object into a target private/public bucket, then remove the temp object.
|
|
* @param {string} tempFilename
|
|
* @param {string} destinationKey
|
|
* @param {string} [targetBucketKind='certificates'] - 'certificates' | 'documents' | bucket name
|
|
*/
|
|
const commitTempFile = async (tempFilename, destinationKey = null, targetBucketKind = 'certificates') => {
|
|
const targetKey = destinationKey || tempFilename;
|
|
const targetBucket = resolveBucket(targetBucketKind);
|
|
|
|
try {
|
|
const client = getS3Client();
|
|
|
|
const copyCommand = new CopyObjectCommand({
|
|
CopySource: `${config.S3_TEMP_BUCKET}/${tempFilename}`,
|
|
Bucket: targetBucket,
|
|
Key: targetKey
|
|
});
|
|
await client.send(copyCommand);
|
|
|
|
const deleteCommand = new DeleteObjectCommand({
|
|
Bucket: config.S3_TEMP_BUCKET,
|
|
Key: tempFilename
|
|
});
|
|
await client.send(deleteCommand);
|
|
|
|
const fileUrl = buildPublicUrl(targetBucket, targetKey);
|
|
logger.info(`[S3 Storage] Committed ${tempFilename} → ${targetBucket}/${targetKey}`);
|
|
return { fileKey: targetKey, bucket: targetBucket, fileUrl };
|
|
} catch (error) {
|
|
logS3Error(`Failed to commit temp file ${tempFilename}`, error);
|
|
if (config.NODE_ENV === 'test' || error.code === 'ECONNREFUSED') {
|
|
logger.warn(`[S3 Storage MOCK] Simulated file commit for ${targetKey}`);
|
|
return {
|
|
fileKey: targetKey,
|
|
bucket: targetBucket,
|
|
fileUrl: buildPublicUrl(targetBucket, targetKey)
|
|
};
|
|
}
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
const generatePresignedUrl = async (
|
|
filename,
|
|
bucketName = config.S3_CERTIFICATES_BUCKET,
|
|
expiresIn = config.SIGNED_URL_EXPIRES_IN
|
|
) => {
|
|
const bucket = resolveBucket(bucketName);
|
|
try {
|
|
const client = getS3Client();
|
|
const command = new GetObjectCommand({
|
|
Bucket: bucket,
|
|
Key: filename
|
|
});
|
|
|
|
return await getSignedUrl(client, command, { expiresIn });
|
|
} catch (error) {
|
|
logger.error(`[S3 Storage ERROR] Failed to generate presigned URL for ${filename}: ${error.message}`);
|
|
return `${config.S3_ENDPOINT}/${bucket}/${filename}?token=mock_presigned_${Date.now()}`;
|
|
}
|
|
};
|
|
|
|
const deleteFromBucket = async (filename, bucketName = config.S3_CERTIFICATES_BUCKET) => {
|
|
const bucket = resolveBucket(bucketName);
|
|
try {
|
|
const client = getS3Client();
|
|
const command = new DeleteObjectCommand({
|
|
Bucket: bucket,
|
|
Key: filename
|
|
});
|
|
await client.send(command);
|
|
logger.info(`[S3 Storage] Deleted file ${filename} from bucket ${bucket}`);
|
|
return true;
|
|
} catch (error) {
|
|
logger.error(`[S3 Storage ERROR] Failed to delete file ${filename}: ${error.message}`);
|
|
return false;
|
|
}
|
|
};
|
|
|
|
const cleanupTempBucket = async (olderThanMinutes = 10) => {
|
|
try {
|
|
const client = getS3Client();
|
|
const listCommand = new ListObjectsV2Command({
|
|
Bucket: config.S3_TEMP_BUCKET
|
|
});
|
|
|
|
const listResult = await client.send(listCommand);
|
|
if (!listResult.Contents || listResult.Contents.length === 0) {
|
|
logger.info('[S3 Temp Cleanup] Temp bucket is empty. Nothing to clean.');
|
|
return 0;
|
|
}
|
|
|
|
const cutoffTime = new Date(Date.now() - olderThanMinutes * 60 * 1000);
|
|
let deletedCount = 0;
|
|
|
|
for (const object of listResult.Contents) {
|
|
if (object.LastModified && new Date(object.LastModified) < cutoffTime) {
|
|
await deleteFromBucket(object.Key, config.S3_TEMP_BUCKET);
|
|
deletedCount++;
|
|
logger.info(`[S3 Temp Cleanup] Deleted expired temp file: ${object.Key}`);
|
|
}
|
|
}
|
|
|
|
logger.info(`[S3 Temp Cleanup] Removed ${deletedCount} expired temp files.`);
|
|
return deletedCount;
|
|
} catch (error) {
|
|
logger.error(`[S3 Temp Cleanup ERROR] Cleanup job failed: ${error.message}`);
|
|
return 0;
|
|
}
|
|
};
|
|
|
|
module.exports = {
|
|
getS3Client,
|
|
uploadToTempBucket,
|
|
commitTempFile,
|
|
generatePresignedUrl,
|
|
deleteFromBucket,
|
|
cleanupTempBucket,
|
|
resolveBucket,
|
|
isBucketPublic,
|
|
buildPublicUrl,
|
|
BUCKETS
|
|
};
|