mirror of
https://github.com/Terranom674/Piwigo_Bratonien_Tools.git
synced 2026-09-19 12:54:31 +00:00
Run manual Piwigo cache build with four workers
This commit is contained in:
Terranom674
@@ -23,7 +23,7 @@ $_SERVER['REQUEST_URI'] = '/';
|
||||
$_SERVER['SCRIPT_NAME'] = '/plugins/bratonien_tools/main-cache-build.php';
|
||||
$_SERVER['PHP_SELF'] = $_SERVER['SCRIPT_NAME'];
|
||||
$_SERVER['QUERY_STRING'] = '';
|
||||
$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.0';
|
||||
$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.1';
|
||||
$_SERVER['HTTPS'] = 'off';
|
||||
|
||||
require_once(PHPWG_ROOT_PATH.'include/common.inc.php');
|
||||
@@ -38,9 +38,9 @@ require_once(BRATONIEN_TOOLS_PATH.'include/watermark_engine.inc.php');
|
||||
require_once(PHPWG_ROOT_PATH.'include/derivative.inc.php');
|
||||
require_once(PHPWG_ROOT_PATH.'admin/include/image.class.php');
|
||||
|
||||
function bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, $current='')
|
||||
function bratonien_tools_cache_builder_status($file, $state, $message, $total=0, $completed=0, $generated=0, $cached=0, $skipped=0, $errors=0, $current='')
|
||||
{
|
||||
bratonien_tools_write_main_cache_status(array(
|
||||
$payload = array(
|
||||
'state'=>$state,
|
||||
'message'=>$message,
|
||||
'total'=>(int)$total,
|
||||
@@ -50,7 +50,14 @@ function bratonien_tools_cache_builder_status($state, $message, $total, $complet
|
||||
'skipped'=>(int)$skipped,
|
||||
'errors'=>(int)$errors,
|
||||
'current'=>(string)$current,
|
||||
));
|
||||
'updated_at'=>time(),
|
||||
);
|
||||
$json = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
|
||||
if ($json === false || @file_put_contents($file, $json."\n", LOCK_EX) === false)
|
||||
{
|
||||
throw new RuntimeException('Worker-Status konnte nicht gespeichert werden.');
|
||||
}
|
||||
@chmod($file, 0664);
|
||||
}
|
||||
|
||||
function bratonien_tools_cache_builder_size($value)
|
||||
@@ -129,15 +136,9 @@ function bratonien_tools_cache_builder_effective_params_from_path($path)
|
||||
|
||||
function bratonien_tools_cache_builder_metadata(array $image, $source_path)
|
||||
{
|
||||
$rotation = 0;
|
||||
if (isset($image['rotation']))
|
||||
{
|
||||
$rotation = pwg_image::get_rotation_angle_from_code($image['rotation']);
|
||||
}
|
||||
else
|
||||
{
|
||||
$rotation = pwg_image::get_rotation_angle($source_path);
|
||||
}
|
||||
$rotation = isset($image['rotation'])
|
||||
? pwg_image::get_rotation_angle_from_code($image['rotation'])
|
||||
: pwg_image::get_rotation_angle($source_path);
|
||||
return array('rotation'=>$rotation, 'coi'=>$image['coi'] ?? null);
|
||||
}
|
||||
|
||||
@@ -179,6 +180,222 @@ function bratonien_tools_cache_builder_generate($source_path, $target_path, $par
|
||||
return is_file($target_path) && is_readable($target_path);
|
||||
}
|
||||
|
||||
function bratonien_tools_cache_builder_variants()
|
||||
{
|
||||
$variants = array();
|
||||
foreach (ImageStdParams::get_defined_type_map() as $type => $params)
|
||||
{
|
||||
$variants['standard:'.$type] = $params;
|
||||
}
|
||||
foreach (ImageStdParams::$custom as $custom_key => $last_used)
|
||||
{
|
||||
$params = bratonien_tools_cache_builder_custom_params($custom_key);
|
||||
if ($params)
|
||||
{
|
||||
$variants['custom:'.$custom_key] = $params;
|
||||
}
|
||||
}
|
||||
return $variants;
|
||||
}
|
||||
|
||||
function bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id)
|
||||
{
|
||||
$status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$worker_index.'.json';
|
||||
$variants = bratonien_tools_cache_builder_variants();
|
||||
$rows = array();
|
||||
$result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id');
|
||||
while ($row = pwg_db_fetch_assoc($result))
|
||||
{
|
||||
if (((int)$row['id'] % $worker_count) === $worker_index)
|
||||
{
|
||||
$rows[] = $row;
|
||||
}
|
||||
}
|
||||
|
||||
$total = count($rows) * count($variants);
|
||||
$completed = 0;
|
||||
$generated = 0;
|
||||
$cached = 0;
|
||||
$skipped = 0;
|
||||
$errors = 0;
|
||||
$last_write = 0.0;
|
||||
|
||||
bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, 0, 0, 0, 0, 0, 'Vorbereitung');
|
||||
|
||||
foreach ($rows as $image_row)
|
||||
{
|
||||
$src = new SrcImage($image_row);
|
||||
$source_path = $src->get_path();
|
||||
$image_id = (int)$image_row['id'];
|
||||
$metadata = null;
|
||||
|
||||
foreach ($variants as $variant_name => $requested_params)
|
||||
{
|
||||
$completed++;
|
||||
$current = 'Worker '.($worker_index + 1).' · Bild #'.$image_id.' · '.$variant_name;
|
||||
|
||||
try
|
||||
{
|
||||
$derivative = new DerivativeImage($requested_params, $src);
|
||||
$target_path = $derivative->get_path();
|
||||
if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0)
|
||||
{
|
||||
$skipped++;
|
||||
}
|
||||
else
|
||||
{
|
||||
$effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path);
|
||||
if (!$effective_params)
|
||||
{
|
||||
$skipped++;
|
||||
}
|
||||
elseif (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time)
|
||||
{
|
||||
$cached++;
|
||||
}
|
||||
elseif (!is_file($source_path) || !is_readable($source_path))
|
||||
{
|
||||
$errors++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($metadata === null)
|
||||
{
|
||||
$metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path);
|
||||
}
|
||||
if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata))
|
||||
{
|
||||
$generated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
$errors++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Throwable $e)
|
||||
{
|
||||
$errors++;
|
||||
fwrite(STDERR, $current.': '.$e->getMessage()."\n");
|
||||
}
|
||||
|
||||
$now = microtime(true);
|
||||
if (($now - $last_write) >= 0.3 || $completed >= $total)
|
||||
{
|
||||
bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, $completed, $generated, $cached, $skipped, $errors, $current);
|
||||
$last_write = $now;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$state = $errors > 0 ? 'error' : 'complete';
|
||||
bratonien_tools_cache_builder_status(
|
||||
$status_file,
|
||||
$state,
|
||||
$errors > 0 ? 'Worker '.($worker_index + 1).' mit Fehlern beendet.' : 'Worker '.($worker_index + 1).' beendet.',
|
||||
$total, $completed, $generated, $cached, $skipped, $errors, ''
|
||||
);
|
||||
exit($errors > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
function bratonien_tools_cache_builder_read_worker($file)
|
||||
{
|
||||
if (!is_file($file) || !is_readable($file))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
$raw = @file_get_contents($file);
|
||||
$data = $raw !== false ? json_decode($raw, true) : null;
|
||||
return is_array($data) ? $data : null;
|
||||
}
|
||||
|
||||
function bratonien_tools_cache_builder_aggregate($run_id, $worker_count, $message)
|
||||
{
|
||||
$sum = array('total'=>0,'completed'=>0,'generated'=>0,'cached'=>0,'skipped'=>0,'errors'=>0);
|
||||
$current = array();
|
||||
$states = array();
|
||||
|
||||
for ($i=0; $i<$worker_count; $i++)
|
||||
{
|
||||
$file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json';
|
||||
$data = bratonien_tools_cache_builder_read_worker($file);
|
||||
if (!$data)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
$states[] = (string)($data['state'] ?? 'running');
|
||||
foreach ($sum as $key => $value)
|
||||
{
|
||||
$sum[$key] += (int)($data[$key] ?? 0);
|
||||
}
|
||||
if (!empty($data['current']))
|
||||
{
|
||||
$current[] = $data['current'];
|
||||
}
|
||||
}
|
||||
|
||||
$state = 'running';
|
||||
if (count($states) === $worker_count)
|
||||
{
|
||||
$all_done = true;
|
||||
foreach ($states as $worker_state)
|
||||
{
|
||||
if ($worker_state === 'running' || $worker_state === 'queued')
|
||||
{
|
||||
$all_done = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if ($all_done)
|
||||
{
|
||||
$state = $sum['errors'] > 0 || in_array('error', $states, true) ? 'error' : 'complete';
|
||||
}
|
||||
}
|
||||
|
||||
bratonien_tools_write_main_cache_status(array(
|
||||
'state'=>$state,
|
||||
'message'=>$message,
|
||||
'total'=>$sum['total'],
|
||||
'completed'=>$sum['completed'],
|
||||
'generated'=>$sum['generated'],
|
||||
'cached'=>$sum['cached'],
|
||||
'skipped'=>$sum['skipped'],
|
||||
'errors'=>$sum['errors'],
|
||||
'current'=>implode(' | ', array_slice($current, 0, 4)),
|
||||
));
|
||||
return $state;
|
||||
}
|
||||
|
||||
$worker_index = null;
|
||||
$worker_count = 4;
|
||||
$run_id = '';
|
||||
foreach ($argv as $arg)
|
||||
{
|
||||
if (preg_match('/^--worker=(\d+)$/', $arg, $m))
|
||||
{
|
||||
$worker_index = (int)$m[1];
|
||||
}
|
||||
elseif (preg_match('/^--workers=(\d+)$/', $arg, $m))
|
||||
{
|
||||
$worker_count = max(1, (int)$m[1]);
|
||||
}
|
||||
elseif (preg_match('/^--run=([A-Za-z0-9_-]+)$/', $arg, $m))
|
||||
{
|
||||
$run_id = $m[1];
|
||||
}
|
||||
}
|
||||
|
||||
if ($worker_index !== null)
|
||||
{
|
||||
if ($run_id === '' || $worker_index < 0 || $worker_index >= $worker_count)
|
||||
{
|
||||
fwrite(STDERR, "Ungueltige Worker-Parameter.\n");
|
||||
exit(1);
|
||||
}
|
||||
bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id);
|
||||
}
|
||||
|
||||
$lock_path = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.lock';
|
||||
$lock = @fopen($lock_path, 'c');
|
||||
if (!$lock || !flock($lock, LOCK_EX | LOCK_NB))
|
||||
@@ -191,121 +408,92 @@ if (!$lock || !flock($lock, LOCK_EX | LOCK_NB))
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// Solange die Bratonien-Engine aktiv ist, muss Piwigos natives Wasserzeichen
|
||||
// fuer alle Derivate neutral bleiben. Der manuelle Cache-Builder erzeugt nur
|
||||
// Piwigos normalen, unveraenderten Derivatcache.
|
||||
bratonien_tools_watermark_engine_enabled();
|
||||
|
||||
$variants = array();
|
||||
foreach (ImageStdParams::get_defined_type_map() as $type => $params)
|
||||
if (!function_exists('proc_open'))
|
||||
{
|
||||
$variants['standard:'.$type] = $params;
|
||||
bratonien_tools_write_main_cache_status(array('state'=>'error','message'=>'proc_open() ist deaktiviert; vier parallele Worker koennen nicht gestartet werden.','errors'=>1));
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
exit(1);
|
||||
}
|
||||
foreach (ImageStdParams::$custom as $custom_key => $last_used)
|
||||
|
||||
$run_id = date('YmdHis').'-'.getmypid();
|
||||
$log = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.log';
|
||||
$processes = array();
|
||||
|
||||
for ($i=0; $i<$worker_count; $i++)
|
||||
{
|
||||
$params = bratonien_tools_cache_builder_custom_params($custom_key);
|
||||
if ($params)
|
||||
$status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json';
|
||||
@unlink($status_file);
|
||||
bratonien_tools_cache_builder_status($status_file, 'queued', 'Worker '.($i + 1).' wartet.', 0, 0, 0, 0, 0, 0, '');
|
||||
|
||||
$command = escapeshellarg(PHP_BINARY).' '.escapeshellarg(__FILE__).' --worker='.$i.' --workers='.$worker_count.' --run='.escapeshellarg($run_id);
|
||||
$descriptors = array(
|
||||
0=>array('file','/dev/null','r'),
|
||||
1=>array('file',$log,'a'),
|
||||
2=>array('file',$log,'a'),
|
||||
);
|
||||
$process = @proc_open($command, $descriptors, $pipes);
|
||||
if (!is_resource($process))
|
||||
{
|
||||
$variants['custom:'.$custom_key] = $params;
|
||||
bratonien_tools_cache_builder_status($status_file, 'error', 'Worker '.($i + 1).' konnte nicht gestartet werden.', 0, 0, 0, 0, 0, 1, '');
|
||||
}
|
||||
else
|
||||
{
|
||||
$processes[$i] = $process;
|
||||
}
|
||||
}
|
||||
|
||||
$image_count = (int)pwg_db_fetch_assoc(pwg_query('SELECT COUNT(*) AS cnt FROM '.IMAGES_TABLE))['cnt'];
|
||||
$total = $image_count * count($variants);
|
||||
$completed = 0;
|
||||
$generated = 0;
|
||||
$cached = 0;
|
||||
$skipped = 0;
|
||||
$errors = 0;
|
||||
$last_write = 0.0;
|
||||
bratonien_tools_write_main_cache_status(array(
|
||||
'state'=>'running',
|
||||
'message'=>'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.',
|
||||
'current'=>'Worker werden gestartet.',
|
||||
));
|
||||
|
||||
bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, 0, 0, 0, 0, 0, 'Vorbereitung');
|
||||
|
||||
$result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id');
|
||||
while ($image_row = pwg_db_fetch_assoc($result))
|
||||
while (true)
|
||||
{
|
||||
$src = new SrcImage($image_row);
|
||||
$source_path = $src->get_path();
|
||||
$image_id = (int)$image_row['id'];
|
||||
$metadata = null;
|
||||
|
||||
foreach ($variants as $variant_name => $requested_params)
|
||||
$running = false;
|
||||
foreach ($processes as $process)
|
||||
{
|
||||
$completed++;
|
||||
$current = 'Bild #'.$image_id.' · '.$variant_name;
|
||||
|
||||
try
|
||||
$info = proc_get_status($process);
|
||||
if (!empty($info['running']))
|
||||
{
|
||||
$derivative = new DerivativeImage($requested_params, $src);
|
||||
$target_path = $derivative->get_path();
|
||||
|
||||
// Kleine Originale koennen bei Piwigo direkt auf das Original oder auf
|
||||
// eine bereits kleinere Standardgroesse zurueckfallen. Dann gibt es fuer
|
||||
// diese Anforderung nichts zusaetzlich zu erzeugen.
|
||||
if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0)
|
||||
{
|
||||
$skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path);
|
||||
if (!$effective_params)
|
||||
{
|
||||
$skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time)
|
||||
{
|
||||
$cached++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!is_file($source_path) || !is_readable($source_path))
|
||||
{
|
||||
$errors++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if ($metadata === null)
|
||||
{
|
||||
$metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path);
|
||||
}
|
||||
if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata))
|
||||
{
|
||||
$generated++;
|
||||
}
|
||||
else
|
||||
{
|
||||
$errors++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Throwable $e)
|
||||
{
|
||||
$errors++;
|
||||
fwrite(STDERR, $current.': '.$e->getMessage()."\n");
|
||||
$running = true;
|
||||
}
|
||||
}
|
||||
|
||||
$now = microtime(true);
|
||||
if (($now - $last_write) >= 0.3 || $completed >= $total)
|
||||
{
|
||||
bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, $completed, $generated, $cached, $skipped, $errors, $current);
|
||||
$last_write = $now;
|
||||
}
|
||||
$state = bratonien_tools_cache_builder_aggregate($run_id, $worker_count, 'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.');
|
||||
if (!$running && ($state === 'complete' || $state === 'error'))
|
||||
{
|
||||
break;
|
||||
}
|
||||
usleep(500000);
|
||||
}
|
||||
|
||||
$exit_error = false;
|
||||
foreach ($processes as $process)
|
||||
{
|
||||
$code = proc_close($process);
|
||||
if ($code !== 0 && $code !== -1)
|
||||
{
|
||||
$exit_error = true;
|
||||
}
|
||||
}
|
||||
|
||||
$state = $errors > 0 ? 'error' : 'complete';
|
||||
$message = $errors > 0
|
||||
? 'Piwigo-Bildcache wurde aufgebaut, einige Varianten konnten jedoch nicht erzeugt werden.'
|
||||
: 'Piwigo-Bildcache wurde vollstaendig aufgebaut.';
|
||||
bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, '');
|
||||
$state = bratonien_tools_cache_builder_aggregate(
|
||||
$run_id,
|
||||
$worker_count,
|
||||
$exit_error ? 'Piwigo-Bildcache beendet; mindestens ein Worker meldete einen Prozessfehler.' : 'Piwigo-Bildcache wurde aufgebaut.'
|
||||
);
|
||||
|
||||
for ($i=0; $i<$worker_count; $i++)
|
||||
{
|
||||
@unlink(PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json');
|
||||
}
|
||||
|
||||
flock($lock, LOCK_UN);
|
||||
fclose($lock);
|
||||
@unlink($lock_path);
|
||||
|
||||
printf("Piwigo-Bildcache: %d/%d, %d neu, %d vorhanden, %d uebersprungen, %d Fehler.\n", $completed, $total, $generated, $cached, $skipped, $errors);
|
||||
exit($errors > 0 ? 1 : 0);
|
||||
exit(($state === 'error' || $exit_error) ? 1 : 0);
|
||||
|
||||
Reference in New Issue
Block a user