342 lines
15 KiB
PHP
342 lines
15 KiB
PHP
<?php
|
||
/**
|
||
* n8n S3 Migration Endpoint
|
||
*
|
||
* Этот скрипт предназначен для вызова из n8n через SSH
|
||
* для автоматической миграции новых файлов в S3
|
||
*/
|
||
|
||
// Параметры по умолчанию
|
||
$defaults = [
|
||
'limit' => 20,
|
||
'dry_run' => 0
|
||
];
|
||
|
||
// Получение параметров из командной строки или переменных окружения
|
||
$limit = isset($argv[1]) ? (int)$argv[1] : (int)($_ENV['LIMIT'] ?? $defaults['limit']);
|
||
$dryRun = isset($argv[2]) ? (int)$argv[2] : (int)($_ENV['DRY_RUN'] ?? $defaults['dry_run']);
|
||
|
||
// Валидация параметров
|
||
$limit = max(1, $limit); // Минимум 1 файл
|
||
$dryRun = $dryRun ? 1 : 0;
|
||
|
||
// Логирование
|
||
$logFile = '/var/www/fastuser/data/www/crm.clientright.ru/logs/n8n_s3_migration.log';
|
||
$logDir = dirname($logFile);
|
||
if (!is_dir($logDir)) {
|
||
mkdir($logDir, 0777, true);
|
||
}
|
||
|
||
function logMessage($message, $toConsole = true) {
|
||
global $logFile;
|
||
$timestamp = date('[Y-m-d H:i:s] ');
|
||
file_put_contents($logFile, $timestamp . $message . "\n", FILE_APPEND);
|
||
if ($toConsole) {
|
||
echo $timestamp . $message . "\n";
|
||
}
|
||
}
|
||
|
||
logMessage("=== n8n S3 Migration Started ===");
|
||
logMessage("Parameters: limit=$limit, dry_run=$dryRun");
|
||
|
||
// Подключение к базе данных
|
||
require_once '/var/www/fastuser/data/www/crm.clientright.ru/config.inc.php';
|
||
$mysqli = new mysqli($dbconfig['db_server'], $dbconfig['db_username'], $dbconfig['db_password'], $dbconfig['db_name']);
|
||
if ($mysqli->connect_error) {
|
||
logMessage("Database connection failed: " . $mysqli->connect_error);
|
||
exit(1);
|
||
}
|
||
$mysqli->set_charset("utf8");
|
||
|
||
// Подключение S3 сервиса
|
||
require_once '/var/www/fastuser/data/www/crm.clientright.ru/include/Storage/S3StorageService.php';
|
||
$s3Service = new S3StorageService();
|
||
$s3Bucket = 'f9825c87-4e3558f6-f9b6-405c-ad3d-d1535c49b61c';
|
||
|
||
// Поиск локальных файлов без S3 метаданных (исключая очищенные файлы)
|
||
$query = "SELECT n.notesid, n.filename, n.filesize, n.filetype, n.filelocationtype,
|
||
a.attachmentsid, a.path, a.storedname
|
||
FROM vtiger_notes n
|
||
INNER JOIN vtiger_seattachmentsrel s ON s.crmid = n.notesid
|
||
INNER JOIN vtiger_attachments a ON a.attachmentsid = s.attachmentsid
|
||
WHERE (n.s3_key IS NULL OR n.s3_key = '')
|
||
AND n.filelocationtype = 'I'
|
||
AND NOT (n.filename IS NULL OR n.filename = '')
|
||
AND NOT (n.filesize = 0)
|
||
AND NOT (n.filename LIKE 'file_15_%')
|
||
AND NOT (n.filename = '7 заявление потребителя')
|
||
ORDER BY n.notesid ASC
|
||
LIMIT ?";
|
||
|
||
$stmt = $mysqli->prepare($query);
|
||
if (!$stmt) {
|
||
logMessage("Prepare failed: " . $mysqli->error);
|
||
exit(1);
|
||
}
|
||
|
||
$stmt->bind_param('i', $limit);
|
||
$stmt->execute();
|
||
$result = $stmt->get_result();
|
||
|
||
$totalProcessed = 0;
|
||
$successfullyMigrated = 0;
|
||
$failedMigrations = 0;
|
||
$skippedFiles = 0;
|
||
|
||
if ($result->num_rows > 0) {
|
||
logMessage("Found " . $result->num_rows . " local files for migration");
|
||
|
||
while ($row = $result->fetch_assoc()) {
|
||
$totalProcessed++;
|
||
$notesid = $row['notesid'];
|
||
$filename = $row['filename'];
|
||
$attachmentsid = $row['attachmentsid'];
|
||
$path = $row['path'];
|
||
$storedname = $row['storedname'];
|
||
|
||
logMessage("Processing file ID $notesid: $filename ({$row['filesize']} bytes)");
|
||
logMessage(" Attachment ID: $attachmentsid, Path: $path, Stored: $storedname");
|
||
|
||
// Поиск локального файла (используем данные из vtiger_attachments)
|
||
$storageDir = '/var/www/fastuser/data/www/crm.clientright.ru/';
|
||
$localFilePath = null;
|
||
|
||
// Основной путь из vtiger_attachments
|
||
$mainPath = $storageDir . $path . $attachmentsid . '_' . $storedname;
|
||
if (file_exists($mainPath) && is_readable($mainPath)) {
|
||
$localFilePath = $mainPath;
|
||
logMessage(" ✅ Found file using attachment data: $mainPath");
|
||
}
|
||
|
||
// Основные пути для поиска (приоритетные)
|
||
$priorityPaths = [
|
||
$storageDir . $filename, // Прямо в корне storage
|
||
$storageDir . $notesid . '_' . $filename, // С префиксом notesid
|
||
];
|
||
|
||
// Дополнительный поиск по имени файла (без ID)
|
||
$cleanFilename = $filename;
|
||
if (preg_match('/^\d+_(.+)$/', $filename, $matches)) {
|
||
$cleanFilename = $matches[1];
|
||
$priorityPaths[] = $storageDir . $cleanFilename;
|
||
}
|
||
|
||
// Проверяем приоритетные пути
|
||
foreach ($priorityPaths as $path) {
|
||
if (file_exists($path) && is_readable($path)) {
|
||
$localFilePath = $path;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// Если не найден в приоритетных путях, ищем в структурированном хранилище
|
||
if ($localFilePath === null) {
|
||
// Поиск по годам (только текущий и предыдущий)
|
||
$currentYear = date('Y');
|
||
for ($year = $currentYear - 1; $year <= $currentYear; $year++) {
|
||
$yearPath = $storageDir . $year . '/';
|
||
if (!is_dir($yearPath)) continue;
|
||
|
||
// Поиск по месяцам (только последние 3 месяца)
|
||
$months = ['January', 'February', 'March', 'April', 'May', 'June',
|
||
'July', 'August', 'September', 'October', 'November', 'December'];
|
||
$currentMonth = (int)date('n') - 1; // 0-based
|
||
|
||
for ($i = 0; $i < 3; $i++) {
|
||
$monthIndex = ($currentMonth - $i + 12) % 12;
|
||
$month = $months[$monthIndex];
|
||
$monthPath = $yearPath . $month . '/';
|
||
|
||
if (!is_dir($monthPath)) continue;
|
||
|
||
// Проверяем основные варианты в месяце
|
||
$monthPaths = [
|
||
$monthPath . $filename,
|
||
$monthPath . $notesid . '_' . $filename,
|
||
];
|
||
|
||
// Добавляем поиск по чистому имени файла
|
||
if (isset($cleanFilename) && $cleanFilename !== $filename) {
|
||
$monthPaths[] = $monthPath . $cleanFilename;
|
||
}
|
||
|
||
// Поиск по близким ID (±10 от текущего)
|
||
for ($idOffset = -10; $idOffset <= 10; $idOffset++) {
|
||
$searchId = $notesid + $idOffset;
|
||
$monthPaths[] = $monthPath . $searchId . '_' . $filename;
|
||
if (isset($cleanFilename) && $cleanFilename !== $filename) {
|
||
$monthPaths[] = $monthPath . $searchId . '_' . $cleanFilename;
|
||
}
|
||
}
|
||
|
||
foreach ($monthPaths as $path) {
|
||
if (file_exists($path) && is_readable($path)) {
|
||
$localFilePath = $path;
|
||
logMessage(" ✅ Found file in structured storage: $path");
|
||
break 3; // Выходим из всех циклов
|
||
}
|
||
}
|
||
|
||
// Поиск по неделям (все 4 недели)
|
||
for ($week = 1; $week <= 4; $week++) {
|
||
$weekPath = $monthPath . 'week' . $week . '/';
|
||
if (!is_dir($weekPath)) continue;
|
||
|
||
$weekPaths = [
|
||
$weekPath . $filename,
|
||
$weekPath . $notesid . '_' . $filename,
|
||
];
|
||
|
||
foreach ($weekPaths as $path) {
|
||
if (file_exists($path) && is_readable($path)) {
|
||
$localFilePath = $path;
|
||
break 4; // Выходим из всех циклов
|
||
}
|
||
}
|
||
|
||
// Поиск по близким ID в неделях
|
||
for ($idOffset = -10; $idOffset <= 10; $idOffset++) {
|
||
$searchId = $notesid + $idOffset;
|
||
$closeIdPath = $weekPath . $searchId . '_' . $filename;
|
||
if (file_exists($closeIdPath) && is_readable($closeIdPath)) {
|
||
$localFilePath = $closeIdPath;
|
||
break 4;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// Проверка существования файла
|
||
if ($localFilePath === null) {
|
||
logMessage(" ⚠️ Physical file not found anywhere. Marking as problematic.");
|
||
|
||
if (!$dryRun) {
|
||
// Помечаем файл как проблемный (добавляем префикс к имени)
|
||
$problematic_filename = "[PROBLEM_" . date('Y-m-d_H-i-s') . "] " . $filename;
|
||
$update_query = "UPDATE vtiger_notes SET filename = ? WHERE notesid = ?";
|
||
$update_stmt = $mysqli->prepare($update_query);
|
||
if ($update_stmt) {
|
||
$update_stmt->bind_param('si', $problematic_filename, $notesid);
|
||
if ($update_stmt->execute()) {
|
||
logMessage(" 🏷️ Marked as problematic: ID $notesid -> $problematic_filename");
|
||
$skippedFiles++; // Считаем как пропущенный
|
||
} else {
|
||
logMessage(" ❌ Failed to mark as problematic for ID $notesid: " . $update_stmt->error);
|
||
$failedMigrations++;
|
||
}
|
||
$update_stmt->close();
|
||
} else {
|
||
logMessage(" ❌ Failed to prepare update query for ID $notesid: " . $mysqli->error);
|
||
$failedMigrations++;
|
||
}
|
||
} else {
|
||
logMessage(" [DRY RUN] Would mark as problematic: ID $notesid");
|
||
$skippedFiles++; // Считаем как пропущенный для dry-run
|
||
}
|
||
continue;
|
||
}
|
||
|
||
if (!$dryRun) {
|
||
try {
|
||
// Загрузка в S3 (используем правильную сигнатуру)
|
||
$s3_result = $s3Service->put($localFilePath, $notesid, $filename);
|
||
|
||
if ($s3_result && isset($s3_result['url'])) {
|
||
$s3_etag = isset($s3_result['etag']) ? trim($s3_result['etag'], '"') : '';
|
||
$s3_url = $s3_result['url'];
|
||
$s3_key = $s3_result['key'];
|
||
|
||
// Обновление базы данных
|
||
$update_query = "UPDATE vtiger_notes SET
|
||
s3_key = ?,
|
||
s3_bucket = ?,
|
||
s3_etag = ?,
|
||
filename = ?,
|
||
filelocationtype = 'E'
|
||
WHERE notesid = ?";
|
||
|
||
$update_stmt = $mysqli->prepare($update_query);
|
||
if (!$update_stmt) {
|
||
logMessage(" ❌ Update prepare failed for $notesid: " . $mysqli->error);
|
||
$failedMigrations++;
|
||
continue;
|
||
}
|
||
|
||
$update_stmt->bind_param('ssssi', $s3_key, $s3Bucket, $s3_etag, $s3_url, $notesid);
|
||
|
||
if ($update_stmt->execute()) {
|
||
logMessage(" ✅ Uploaded to S3 and DB updated. S3 URL: $s3_url");
|
||
$successfullyMigrated++;
|
||
|
||
// Удаление локального файла
|
||
if (unlink($localFilePath)) {
|
||
logMessage(" 🗑️ Local file deleted: $localFilePath");
|
||
} else {
|
||
logMessage(" ⚠️ Warning: Could not delete local file: $localFilePath");
|
||
}
|
||
} else {
|
||
logMessage(" ❌ Failed to update DB for $notesid: " . $update_stmt->error);
|
||
$failedMigrations++;
|
||
}
|
||
$update_stmt->close();
|
||
} else {
|
||
logMessage(" ❌ S3 upload failed for $notesid. Result: " . json_encode($s3_result));
|
||
$failedMigrations++;
|
||
}
|
||
} catch (Exception $e) {
|
||
logMessage(" ❌ S3 upload error for $notesid: " . $e->getMessage());
|
||
$failedMigrations++;
|
||
}
|
||
} else {
|
||
logMessage(" [DRY RUN] Would upload $localFilePath to S3 for notesid $notesid and update DB.");
|
||
$successfullyMigrated++; // Считаем как успешный для dry-run
|
||
}
|
||
}
|
||
} else {
|
||
logMessage("No local files found for migration.");
|
||
}
|
||
|
||
$stmt->close();
|
||
$mysqli->close();
|
||
|
||
// Итоговая статистика
|
||
logMessage("=== Migration Summary ===");
|
||
logMessage("Total files processed: $totalProcessed");
|
||
logMessage("Successfully migrated: $successfullyMigrated");
|
||
logMessage("Failed: $failedMigrations");
|
||
logMessage("Marked as problematic: $skippedFiles");
|
||
logMessage("Dry run: " . ($dryRun ? "YES" : "NO"));
|
||
logMessage("=== n8n S3 Migration Finished ===");
|
||
|
||
// JSON вывод для n8n
|
||
$jsonOutput = [
|
||
'status' => 'success',
|
||
'timestamp' => date('Y-m-d H:i:s'),
|
||
'summary' => [
|
||
'total_processed' => $totalProcessed,
|
||
'successfully_migrated' => $successfullyMigrated,
|
||
'failed' => $failedMigrations,
|
||
'marked_problematic' => $skippedFiles,
|
||
'dry_run' => $dryRun ? true : false
|
||
],
|
||
'exit_code' => 0
|
||
];
|
||
|
||
// Определяем статус и код выхода
|
||
if ($failedMigrations > 0) {
|
||
$jsonOutput['status'] = 'partial_error';
|
||
$jsonOutput['exit_code'] = 2;
|
||
} elseif ($skippedFiles > 0) {
|
||
$jsonOutput['status'] = 'warning';
|
||
$jsonOutput['exit_code'] = 1;
|
||
}
|
||
|
||
// Выводим JSON
|
||
echo json_encode($jsonOutput, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT) . "\n";
|
||
|
||
// Возвращаем код выхода для n8n
|
||
exit($jsonOutput['exit_code']);
|
||
?>
|