From c9798c287870d8cdcd98694fcbf30ccdc420e941 Mon Sep 17 00:00:00 2001 From: Terranom674 Date: Sat, 15 Aug 2026 19:18:44 +0200 Subject: [PATCH] Run manual Piwigo cache build with four workers --- main-cache-build.php | 404 +++++++++++++++++++++++++++++++------------ 1 file changed, 296 insertions(+), 108 deletions(-) diff --git a/main-cache-build.php b/main-cache-build.php index 788d954..6229ac1 100644 --- a/main-cache-build.php +++ b/main-cache-build.php @@ -23,7 +23,7 @@ $_SERVER['REQUEST_URI'] = '/'; $_SERVER['SCRIPT_NAME'] = '/plugins/bratonien_tools/main-cache-build.php'; $_SERVER['PHP_SELF'] = $_SERVER['SCRIPT_NAME']; $_SERVER['QUERY_STRING'] = ''; -$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.0'; +$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.1'; $_SERVER['HTTPS'] = 'off'; require_once(PHPWG_ROOT_PATH.'include/common.inc.php'); @@ -38,9 +38,9 @@ require_once(BRATONIEN_TOOLS_PATH.'include/watermark_engine.inc.php'); require_once(PHPWG_ROOT_PATH.'include/derivative.inc.php'); require_once(PHPWG_ROOT_PATH.'admin/include/image.class.php'); -function bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, $current='') +function bratonien_tools_cache_builder_status($file, $state, $message, $total=0, $completed=0, $generated=0, $cached=0, $skipped=0, $errors=0, $current='') { - bratonien_tools_write_main_cache_status(array( + $payload = array( 'state'=>$state, 'message'=>$message, 'total'=>(int)$total, @@ -50,7 +50,14 @@ function bratonien_tools_cache_builder_status($state, $message, $total, $complet 'skipped'=>(int)$skipped, 'errors'=>(int)$errors, 'current'=>(string)$current, - )); + 'updated_at'=>time(), + ); + $json = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES); + if ($json === false || @file_put_contents($file, $json."\n", LOCK_EX) === false) + { + throw new RuntimeException('Worker-Status konnte nicht gespeichert werden.'); + } + @chmod($file, 0664); } function bratonien_tools_cache_builder_size($value) @@ -129,15 +136,9 @@ function bratonien_tools_cache_builder_effective_params_from_path($path) function bratonien_tools_cache_builder_metadata(array $image, $source_path) { - $rotation = 0; - if (isset($image['rotation'])) - { - $rotation = pwg_image::get_rotation_angle_from_code($image['rotation']); - } - else - { - $rotation = pwg_image::get_rotation_angle($source_path); - } + $rotation = isset($image['rotation']) + ? pwg_image::get_rotation_angle_from_code($image['rotation']) + : pwg_image::get_rotation_angle($source_path); return array('rotation'=>$rotation, 'coi'=>$image['coi'] ?? null); } @@ -179,6 +180,222 @@ function bratonien_tools_cache_builder_generate($source_path, $target_path, $par return is_file($target_path) && is_readable($target_path); } +function bratonien_tools_cache_builder_variants() +{ + $variants = array(); + foreach (ImageStdParams::get_defined_type_map() as $type => $params) + { + $variants['standard:'.$type] = $params; + } + foreach (ImageStdParams::$custom as $custom_key => $last_used) + { + $params = bratonien_tools_cache_builder_custom_params($custom_key); + if ($params) + { + $variants['custom:'.$custom_key] = $params; + } + } + return $variants; +} + +function bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id) +{ + $status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$worker_index.'.json'; + $variants = bratonien_tools_cache_builder_variants(); + $rows = array(); + $result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id'); + while ($row = pwg_db_fetch_assoc($result)) + { + if (((int)$row['id'] % $worker_count) === $worker_index) + { + $rows[] = $row; + } + } + + $total = count($rows) * count($variants); + $completed = 0; + $generated = 0; + $cached = 0; + $skipped = 0; + $errors = 0; + $last_write = 0.0; + + bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, 0, 0, 0, 0, 0, 'Vorbereitung'); + + foreach ($rows as $image_row) + { + $src = new SrcImage($image_row); + $source_path = $src->get_path(); + $image_id = (int)$image_row['id']; + $metadata = null; + + foreach ($variants as $variant_name => $requested_params) + { + $completed++; + $current = 'Worker '.($worker_index + 1).' · Bild #'.$image_id.' · '.$variant_name; + + try + { + $derivative = new DerivativeImage($requested_params, $src); + $target_path = $derivative->get_path(); + if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0) + { + $skipped++; + } + else + { + $effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path); + if (!$effective_params) + { + $skipped++; + } + elseif (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time) + { + $cached++; + } + elseif (!is_file($source_path) || !is_readable($source_path)) + { + $errors++; + } + else + { + if ($metadata === null) + { + $metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path); + } + if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata)) + { + $generated++; + } + else + { + $errors++; + } + } + } + } + catch (Throwable $e) + { + $errors++; + fwrite(STDERR, $current.': '.$e->getMessage()."\n"); + } + + $now = microtime(true); + if (($now - $last_write) >= 0.3 || $completed >= $total) + { + bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, $completed, $generated, $cached, $skipped, $errors, $current); + $last_write = $now; + } + } + } + + $state = $errors > 0 ? 'error' : 'complete'; + bratonien_tools_cache_builder_status( + $status_file, + $state, + $errors > 0 ? 'Worker '.($worker_index + 1).' mit Fehlern beendet.' : 'Worker '.($worker_index + 1).' beendet.', + $total, $completed, $generated, $cached, $skipped, $errors, '' + ); + exit($errors > 0 ? 1 : 0); +} + +function bratonien_tools_cache_builder_read_worker($file) +{ + if (!is_file($file) || !is_readable($file)) + { + return null; + } + $raw = @file_get_contents($file); + $data = $raw !== false ? json_decode($raw, true) : null; + return is_array($data) ? $data : null; +} + +function bratonien_tools_cache_builder_aggregate($run_id, $worker_count, $message) +{ + $sum = array('total'=>0,'completed'=>0,'generated'=>0,'cached'=>0,'skipped'=>0,'errors'=>0); + $current = array(); + $states = array(); + + for ($i=0; $i<$worker_count; $i++) + { + $file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json'; + $data = bratonien_tools_cache_builder_read_worker($file); + if (!$data) + { + continue; + } + $states[] = (string)($data['state'] ?? 'running'); + foreach ($sum as $key => $value) + { + $sum[$key] += (int)($data[$key] ?? 0); + } + if (!empty($data['current'])) + { + $current[] = $data['current']; + } + } + + $state = 'running'; + if (count($states) === $worker_count) + { + $all_done = true; + foreach ($states as $worker_state) + { + if ($worker_state === 'running' || $worker_state === 'queued') + { + $all_done = false; + break; + } + } + if ($all_done) + { + $state = $sum['errors'] > 0 || in_array('error', $states, true) ? 'error' : 'complete'; + } + } + + bratonien_tools_write_main_cache_status(array( + 'state'=>$state, + 'message'=>$message, + 'total'=>$sum['total'], + 'completed'=>$sum['completed'], + 'generated'=>$sum['generated'], + 'cached'=>$sum['cached'], + 'skipped'=>$sum['skipped'], + 'errors'=>$sum['errors'], + 'current'=>implode(' | ', array_slice($current, 0, 4)), + )); + return $state; +} + +$worker_index = null; +$worker_count = 4; +$run_id = ''; +foreach ($argv as $arg) +{ + if (preg_match('/^--worker=(\d+)$/', $arg, $m)) + { + $worker_index = (int)$m[1]; + } + elseif (preg_match('/^--workers=(\d+)$/', $arg, $m)) + { + $worker_count = max(1, (int)$m[1]); + } + elseif (preg_match('/^--run=([A-Za-z0-9_-]+)$/', $arg, $m)) + { + $run_id = $m[1]; + } +} + +if ($worker_index !== null) +{ + if ($run_id === '' || $worker_index < 0 || $worker_index >= $worker_count) + { + fwrite(STDERR, "Ungueltige Worker-Parameter.\n"); + exit(1); + } + bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id); +} + $lock_path = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.lock'; $lock = @fopen($lock_path, 'c'); if (!$lock || !flock($lock, LOCK_EX | LOCK_NB)) @@ -191,121 +408,92 @@ if (!$lock || !flock($lock, LOCK_EX | LOCK_NB)) exit(0); } -// Solange die Bratonien-Engine aktiv ist, muss Piwigos natives Wasserzeichen -// fuer alle Derivate neutral bleiben. Der manuelle Cache-Builder erzeugt nur -// Piwigos normalen, unveraenderten Derivatcache. bratonien_tools_watermark_engine_enabled(); -$variants = array(); -foreach (ImageStdParams::get_defined_type_map() as $type => $params) +if (!function_exists('proc_open')) { - $variants['standard:'.$type] = $params; + bratonien_tools_write_main_cache_status(array('state'=>'error','message'=>'proc_open() ist deaktiviert; vier parallele Worker koennen nicht gestartet werden.','errors'=>1)); + flock($lock, LOCK_UN); + fclose($lock); + exit(1); } -foreach (ImageStdParams::$custom as $custom_key => $last_used) + +$run_id = date('YmdHis').'-'.getmypid(); +$log = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.log'; +$processes = array(); + +for ($i=0; $i<$worker_count; $i++) { - $params = bratonien_tools_cache_builder_custom_params($custom_key); - if ($params) + $status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json'; + @unlink($status_file); + bratonien_tools_cache_builder_status($status_file, 'queued', 'Worker '.($i + 1).' wartet.', 0, 0, 0, 0, 0, 0, ''); + + $command = escapeshellarg(PHP_BINARY).' '.escapeshellarg(__FILE__).' --worker='.$i.' --workers='.$worker_count.' --run='.escapeshellarg($run_id); + $descriptors = array( + 0=>array('file','/dev/null','r'), + 1=>array('file',$log,'a'), + 2=>array('file',$log,'a'), + ); + $process = @proc_open($command, $descriptors, $pipes); + if (!is_resource($process)) { - $variants['custom:'.$custom_key] = $params; + bratonien_tools_cache_builder_status($status_file, 'error', 'Worker '.($i + 1).' konnte nicht gestartet werden.', 0, 0, 0, 0, 0, 1, ''); + } + else + { + $processes[$i] = $process; } } -$image_count = (int)pwg_db_fetch_assoc(pwg_query('SELECT COUNT(*) AS cnt FROM '.IMAGES_TABLE))['cnt']; -$total = $image_count * count($variants); -$completed = 0; -$generated = 0; -$cached = 0; -$skipped = 0; -$errors = 0; -$last_write = 0.0; +bratonien_tools_write_main_cache_status(array( + 'state'=>'running', + 'message'=>'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.', + 'current'=>'Worker werden gestartet.', +)); -bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, 0, 0, 0, 0, 0, 'Vorbereitung'); - -$result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id'); -while ($image_row = pwg_db_fetch_assoc($result)) +while (true) { - $src = new SrcImage($image_row); - $source_path = $src->get_path(); - $image_id = (int)$image_row['id']; - $metadata = null; - - foreach ($variants as $variant_name => $requested_params) + $running = false; + foreach ($processes as $process) { - $completed++; - $current = 'Bild #'.$image_id.' · '.$variant_name; - - try + $info = proc_get_status($process); + if (!empty($info['running'])) { - $derivative = new DerivativeImage($requested_params, $src); - $target_path = $derivative->get_path(); - - // Kleine Originale koennen bei Piwigo direkt auf das Original oder auf - // eine bereits kleinere Standardgroesse zurueckfallen. Dann gibt es fuer - // diese Anforderung nichts zusaetzlich zu erzeugen. - if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0) - { - $skipped++; - continue; - } - - $effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path); - if (!$effective_params) - { - $skipped++; - continue; - } - - if (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time) - { - $cached++; - } - else - { - if (!is_file($source_path) || !is_readable($source_path)) - { - $errors++; - } - else - { - if ($metadata === null) - { - $metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path); - } - if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata)) - { - $generated++; - } - else - { - $errors++; - } - } - } - } - catch (Throwable $e) - { - $errors++; - fwrite(STDERR, $current.': '.$e->getMessage()."\n"); + $running = true; } + } - $now = microtime(true); - if (($now - $last_write) >= 0.3 || $completed >= $total) - { - bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, $completed, $generated, $cached, $skipped, $errors, $current); - $last_write = $now; - } + $state = bratonien_tools_cache_builder_aggregate($run_id, $worker_count, 'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.'); + if (!$running && ($state === 'complete' || $state === 'error')) + { + break; + } + usleep(500000); +} + +$exit_error = false; +foreach ($processes as $process) +{ + $code = proc_close($process); + if ($code !== 0 && $code !== -1) + { + $exit_error = true; } } -$state = $errors > 0 ? 'error' : 'complete'; -$message = $errors > 0 - ? 'Piwigo-Bildcache wurde aufgebaut, einige Varianten konnten jedoch nicht erzeugt werden.' - : 'Piwigo-Bildcache wurde vollstaendig aufgebaut.'; -bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, ''); +$state = bratonien_tools_cache_builder_aggregate( + $run_id, + $worker_count, + $exit_error ? 'Piwigo-Bildcache beendet; mindestens ein Worker meldete einen Prozessfehler.' : 'Piwigo-Bildcache wurde aufgebaut.' +); + +for ($i=0; $i<$worker_count; $i++) +{ + @unlink(PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json'); +} flock($lock, LOCK_UN); fclose($lock); @unlink($lock_path); -printf("Piwigo-Bildcache: %d/%d, %d neu, %d vorhanden, %d uebersprungen, %d Fehler.\n", $completed, $total, $generated, $cached, $skipped, $errors); -exit($errors > 0 ? 1 : 0); +exit(($state === 'error' || $exit_error) ? 1 : 0);