<?php
/**
 * nest_backfill.php — one-off historical pull via `POST /records/tenders`.
 * API caps pageSize at 10. Iterates pages 1..N.
 *
 * Usage:
 *   php app/cli/nest_backfill.php --since=2025-07-01T00:00:00 --max-pages=2000
 *   php app/cli/nest_backfill.php --starting-page=500 --max-pages=100
 *
 * `--since` is retained here for the log entry (the /records/tenders endpoint
 *  itself doesn't accept it, but our OCID FY filter applies at upsert-time).
 */
if (PHP_SAPI !== 'cli') { fwrite(STDERR, "CLI only.\n"); exit(2); }
define('IS_CLI', true);
define('DS', DIRECTORY_SEPARATOR);
define('ROOT', dirname(__DIR__, 2));

spl_autoload_register(function ($cls) {
    foreach (['app/models','app/controllers','core'] as $d) {
        $p = ROOT . '/' . $d . '/' . $cls . '.php';
        if (file_exists($p)) { require $p; return; }
    }
});
require ROOT . '/config/config.php';

$opts = getopt('', ['since::','max-pages::','starting-page::','dry-run::']);
$dryRun    = isset($opts['dry-run']);
$maxPages  = isset($opts['max-pages']) ? max(1, (int)$opts['max-pages']) : 100;
$page      = isset($opts['starting-page']) ? max(1, (int)$opts['starting-page']) : 1;
$since     = $opts['since'] ?? null;

$nt = new NestTenders();
$logId = $nt->logStart('bulk', $since, $page);
echo "[" . date('Y-m-d H:i:s') . "] nest_backfill · start-page=$page · max-pages=$maxPages · dry-run=" . ($dryRun ? 'yes' : 'no') . " · log_id=$logId" . PHP_EOL;

$base = 'https://nest.go.tz/gateway/nest-data-portal-api/records/tenders';
$fetched = 0; $inserted = 0; $updated = 0; $skipped = 0; $errors = 0; $skippedUrls = [];
$pagesDone = 0;
$maxRetries = 4;
$pageSize = 10; // API hard cap

while ($pagesDone < $maxPages) {
    $body = null; $retry = 0; $tag = "page=$page";
    while ($retry <= $maxRetries) {
        $ch = curl_init($base);
        curl_setopt_array($ch, [
            CURLOPT_RETURNTRANSFER => true,
            CURLOPT_TIMEOUT        => 180,
            CURLOPT_CONNECTTIMEOUT => 30,
            CURLOPT_USERAGENT      => 'rsflow.dspera/nest-backfill',
            CURLOPT_POST           => true,
            CURLOPT_POSTFIELDS     => json_encode(['page' => $page, 'pageSize' => $pageSize]),
            CURLOPT_HTTPHEADER     => ['Content-Type: application/json', 'Accept: application/json'],
            CURLOPT_SSL_VERIFYPEER => false, // WAMP dev boxes rarely have a CA bundle configured
            CURLOPT_SSL_VERIFYHOST => 0,
        ]);
        $body = curl_exec($ch);
        $code = curl_getinfo($ch, CURLINFO_HTTP_CODE);
        $err  = curl_error($ch);
        curl_close($ch);
        if ($body !== false && $code >= 200 && $code < 300) break;

        $wait = [5, 15, 45, 120][$retry] ?? 120;
        echo "  ! HTTP $code · $tag · retry=$retry · sleeping {$wait}s · " . substr((string)$err, 0, 80) . PHP_EOL;
        $errors++; sleep($wait); $retry++;
    }
    if ($body === false || $body === null) {
        $skippedUrls[] = "$base [$tag]";
        $page++; $pagesDone++;
        continue;
    }

    $decoded = json_decode($body, true);
    // NeST wraps records under `data`; older schemas used `records` or `releases`
    $recs = $decoded['data'] ?? $decoded['records'] ?? $decoded['releases'] ?? [];
    if (!$recs) {
        echo "  · empty $tag — end of pages" . PHP_EOL;
        break;
    }
    echo "  $tag · " . count($recs) . " record(s) · totalPages=" . ($decoded['totalPages'] ?? '?') . PHP_EOL;

    foreach ($recs as $rec) {
        // Each record has ocid at top-level + compiledRelease (a real OCDS release).
        // Prefer compiledRelease (has tender/parties inside); fall back to first release.
        $release = $rec['compiledRelease'] ?? ($rec['releases'][0] ?? null);
        if (!$release || !is_array($release)) { $skipped++; continue; }
        // Backfill ocid + date if the release doesn't carry them
        if (empty($release['ocid']))         $release['ocid']  = (string)($rec['ocid'] ?? '');
        if (empty($release['date']) && !empty($rec['lastReleaseDate'])) $release['date'] = $rec['lastReleaseDate'];
        // Enrich with the top-level facets (FY, category acronym) that live outside compiledRelease
        if (!isset($release['_rsflow_ext']))  $release['_rsflow_ext'] = [];
        $release['_rsflow_ext']['financialYear']         = $rec['financialYear']            ?? null;
        $release['_rsflow_ext']['tenderCategoryAcronym'] = $rec['tenderCategoryAcronym']    ?? null;
        $release['_rsflow_ext']['subCategoryAcronym']    = $rec['tenderSubCategoryAcronym'] ?? null;

        $fetched++;
        if ($dryRun) { $skipped++; continue; }
        try {
            [$outcome] = $nt->upsertFromRelease($release);
            if ($outcome === 'inserted')     $inserted++;
            elseif ($outcome === 'updated')  $updated++;
            else                             $skipped++;
        } catch (Throwable $e) {
            $errors++;
            echo "    err: " . $e->getMessage() . PHP_EOL;
        }
    }
    $page++; $pagesDone++;
}

$nt->logFinish($logId, [
    'fetched' => $fetched, 'inserted' => $inserted, 'updated' => $updated,
    'skipped' => $skipped, 'errors' => $errors,
    'page_end' => $page - 1,
], $errors > 0 && ($inserted + $updated) === 0 ? 'failed' : 'done',
    "start=" . ($opts['starting-page'] ?? '1') . " · pages=$pagesDone", $skippedUrls);

echo "[" . date('Y-m-d H:i:s') . "] done · fetched=$fetched · inserted=$inserted · updated=$updated · skipped=$skipped · errors=$errors · pages=$pagesDone" . PHP_EOL;
exit($errors > 0 && ($inserted + $updated) === 0 ? 1 : 0);
