crm.clientright.ru/crm_extensions/file_storage/migrate_to_s3.php

<?php
/**
 * vTiger → S3 migration (Phase 1): copy-only, no DB writes.
 *
 * - Reads attachments from vtiger tables and uploads files to S3 using S3StorageService
 * - Respects existing storage paths; does NOT touch `cache/`
 * - By default runs in dry-run mode (no S3 calls), writes logs and JSON summary
 * - Intended to be executed in small batches (limit/offset)
 */

ini_set('memory_limit','1024M');
set_time_limit(0);

date_default_timezone_set('Europe/Moscow');

$ROOT = '/var/www/fastuser/data/www/crm.clientright.ru/';

// Dependencies (минимум для dry-run)
require_once $ROOT . 'config.inc.php'; // $dbconfig, $root_directory

// CLI options
$opts = getopt('', [
    'limit::',
    'offset::',
    'dry-run::',
    'only-not-migrated::',
    'max-size::'
]);

$limit = isset($opts['limit']) ? (int)$opts['limit'] : 200;
$offset = isset($opts['offset']) ? (int)$opts['offset'] : 0;
$dryRun = isset($opts['dry-run']) ? (int)$opts['dry-run'] !== 0 : true; // default: dry-run
$onlyNotMigrated = isset($opts['only-not-migrated']) ? (int)$opts['only-not-migrated'] !== 0 : true;
// По умолчанию лимит 100MB; в real-run можно будет переопределить из конфига при необходимости
$maxSize = isset($opts['max-size']) ? (int)$opts['max-size'] : 104857600;

// Logging
$logDir = $ROOT . 'logs';
if (!is_dir($logDir)) mkdir($logDir, 0755, true);
$logFile = $logDir . '/s3_migration.log';
$summaryDir = $ROOT . 'crm_extensions/file_storage';
if (!is_dir($summaryDir)) mkdir($summaryDir, 0755, true);
$summaryFile = $summaryDir . '/migration_results_' . date('Ymd_His') . '.json';

function logln($msg) {
    global $logFile;
    $line = '[' . date('Y-m-d H:i:s') . '] ' . $msg . PHP_EOL;
    file_put_contents($logFile, $line, FILE_APPEND | LOCK_EX);
    echo $line;
}

function humanSize($bytes) {
    $units = ['B','KB','MB','GB','TB'];
    $i = 0;
    while ($bytes >= 1024 && $i < count($units)-1) { $bytes /= 1024; $i++; }
    return sprintf('%.2f %s', $bytes, $units[$i]);
}

// DB connect (PDO)
try {
    $host = $dbconfig['db_server'] ?: 'localhost';
    $dbname = $dbconfig['db_name'];
    $dsn = 'mysql:host=' . $host . ';dbname=' . $dbname . ';charset=utf8';
    $pdo = new PDO($dsn, $dbconfig['db_username'], $dbconfig['db_password'], [
        PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION,
        PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC,
    ]);
} catch (Exception $e) {
    logln('DB connection error: ' . $e->getMessage());
    exit(1);
}

// Check if vtiger_notes.s3_key exists (to support only-not-migrated flag)
$s3KeyExists = false;
try {
    $cols = $pdo->query("SHOW COLUMNS FROM vtiger_notes LIKE 's3_key'");
    $s3KeyExists = $cols && $cols->rowCount() > 0;
} catch (Exception $e) {
    // ignore
}
if ($onlyNotMigrated && !$s3KeyExists) {
    logln("Column vtiger_notes.s3_key not found — disabling only-not-migrated filter");
    $onlyNotMigrated = false;
}

$conditions = ["n.filelocationtype = 'I'"]; // internal storage only
if ($onlyNotMigrated) {
    $conditions[] = "(n.s3_key IS NULL OR n.s3_key = '')";
}
$where = implode(' AND ', $conditions);

$sql = "
SELECT
    n.notesid,
    n.filename AS original_name,
    a.attachmentsid,
    a.name AS attachment_name,
    a.type AS mime_type,
    a.path,
    a.storedname
FROM vtiger_notes n
JOIN vtiger_seattachmentsrel r ON r.crmid = n.notesid
JOIN vtiger_attachments a ON a.attachmentsid = r.attachmentsid
WHERE $where
ORDER BY n.notesid ASC
LIMIT :limit OFFSET :offset
";

$stmt = $pdo->prepare($sql);
$stmt->bindValue(':limit', $limit, PDO::PARAM_INT);
$stmt->bindValue(':offset', $offset, PDO::PARAM_INT);
$stmt->execute();
$rows = $stmt->fetchAll();

logln("Rows: " . count($rows) . " (limit=$limit, offset=$offset, dryRun=" . ($dryRun ? 'true' : 'false') . ", onlyNotMigrated=" . ($onlyNotMigrated ? 'true' : 'false') . ")");

// S3 service подключаем ТОЛЬКО в real-run, чтобы не требовался .env в dry-run
$service = null;
$s3Prefix = 'crm2/CRM_Active_Files/Documents';
if (!$dryRun) {
    require_once $ROOT . 'include/Storage/S3StorageService.php';
    $service = new S3StorageService();
    $s3Prefix = $service->getPrefix();
}

$results = [
    'started_at' => date('c'),
    'dry_run' => $dryRun,
    'limit' => $limit,
    'offset' => $offset,
    'copied' => [],
    'skipped' => [],
    'errors' => []
];

foreach ($rows as $row) {
    $notesId = (int)$row['notesid'];
    $originalName = (string)$row['original_name'];
    $attachmentsId = (int)$row['attachmentsid'];
    $path = (string)$row['path'];
    $storedname = isset($row['storedname']) && $row['storedname'] !== null && $row['storedname'] !== ''
        ? $row['storedname']
        : $row['attachment_name'];

    $localPath = $root_directory . $path . $attachmentsId . '_' . $storedname;

    if (!file_exists($localPath)) {
        $msg = "MISSING local file: notesId=$notesId, path=$localPath";
        logln($msg);
        $results['skipped'][] = ['notesid'=>$notesId,'reason'=>'missing','local_path'=>$localPath];
        continue;
    }

    $size = @filesize($localPath);
    if ($size === false) { $size = 0; }

    if ($maxSize > 0 && $size > $maxSize) {
        $msg = "SKIP too large: notesId=$notesId, size=" . humanSize($size) . ", path=$localPath";
        logln($msg);
        $results['skipped'][] = ['notesid'=>$notesId,'reason'=>'too_large','size'=>$size,'local_path'=>$localPath];
        continue;
    }

    // Build expected S3 key (same logic as S3StorageService->put)
    $keyPreview = $s3Prefix . '/' . $notesId . '/' . $originalName;

    if ($dryRun) {
        $msg = "DRY-RUN would upload: notesId=$notesId, size=" . humanSize($size) . ", key=$keyPreview";
        logln($msg);
        $results['copied'][] = [
            'notesid'=>$notesId,
            'key'=>$keyPreview,
            'size'=>$size,
            'local_path'=>$localPath,
            'dry_run'=>true
        ];
        continue;
    }

    try {
        $res = $service->put($localPath, $notesId, $originalName);
        $msg = "UPLOADED: notesId=$notesId, size=" . humanSize($size) . ", key=" . ($res['key'] ?? $keyPreview);
        logln($msg);
        $results['copied'][] = [
            'notesid'=>$notesId,
            'key'=>$res['key'] ?? $keyPreview,
            'etag'=>$res['etag'] ?? null,
            'url'=>$res['url'] ?? null,
            'bucket'=>$res['bucket'] ?? null,
            'size'=>$res['size'] ?? $size,
            'mimeType'=>$res['mimeType'] ?? null,
            'local_path'=>$localPath
        ];
    } catch (Exception $e) {
        $msg = "ERROR upload notesId=$notesId: " . $e->getMessage();
        logln($msg);
        $results['errors'][] = [
            'notesid'=>$notesId,
            'error'=>$e->getMessage(),
            'local_path'=>$localPath
        ];
    }
}

$results['finished_at'] = date('c');
file_put_contents($summaryFile, json_encode($results, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT));
logln("Summary saved to: $summaryFile");
logln('Done.');

// EOF