Run manual Piwigo cache build with four workers

main-cache-build.php

@@ -23,7 +23,7 @@ $_SERVER['REQUEST_URI'] = '/';
 $_SERVER['SCRIPT_NAME'] = '/plugins/bratonien_tools/main-cache-build.php';
 $_SERVER['PHP_SELF'] = $_SERVER['SCRIPT_NAME'];
 $_SERVER['QUERY_STRING'] = '';
-$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.0';
+$_SERVER['HTTP_USER_AGENT'] = 'Bratonien-Piwigo-Cache-Builder/1.1';
 $_SERVER['HTTPS'] = 'off';
 
 require_once(PHPWG_ROOT_PATH.'include/common.inc.php');
@@ -38,9 +38,9 @@ require_once(BRATONIEN_TOOLS_PATH.'include/watermark_engine.inc.php');
 require_once(PHPWG_ROOT_PATH.'include/derivative.inc.php');
 require_once(PHPWG_ROOT_PATH.'admin/include/image.class.php');
 
-function bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, $current='')
+function bratonien_tools_cache_builder_status($file, $state, $message, $total=0, $completed=0, $generated=0, $cached=0, $skipped=0, $errors=0, $current='')
 {
-  bratonien_tools_write_main_cache_status(array(
+  $payload = array(
     'state'=>$state,
     'message'=>$message,
     'total'=>(int)$total,
@@ -50,7 +50,14 @@ function bratonien_tools_cache_builder_status($state, $message, $total, $complet
     'skipped'=>(int)$skipped,
     'errors'=>(int)$errors,
     'current'=>(string)$current,
-  ));
+    'updated_at'=>time(),
+  );
+  $json = json_encode($payload, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
+  if ($json === false || @file_put_contents($file, $json."\n", LOCK_EX) === false)
+  {
+    throw new RuntimeException('Worker-Status konnte nicht gespeichert werden.');
+  }
+  @chmod($file, 0664);
 }
 
 function bratonien_tools_cache_builder_size($value)
@@ -129,15 +136,9 @@ function bratonien_tools_cache_builder_effective_params_from_path($path)
 
 function bratonien_tools_cache_builder_metadata(array $image, $source_path)
 {
-  $rotation = 0;
+  $rotation = isset($image['rotation'])
-  if (isset($image['rotation']))
+    ? pwg_image::get_rotation_angle_from_code($image['rotation'])
-  {
+    : pwg_image::get_rotation_angle($source_path);
-    $rotation = pwg_image::get_rotation_angle_from_code($image['rotation']);
-  }
-  else
-  {
-    $rotation = pwg_image::get_rotation_angle($source_path);
-  }
   return array('rotation'=>$rotation, 'coi'=>$image['coi'] ?? null);
 }
 
@@ -179,6 +180,222 @@ function bratonien_tools_cache_builder_generate($source_path, $target_path, $par
   return is_file($target_path) && is_readable($target_path);
 }
 
+function bratonien_tools_cache_builder_variants()
+{
+  $variants = array();
+  foreach (ImageStdParams::get_defined_type_map() as $type => $params)
+  {
+    $variants['standard:'.$type] = $params;
+  }
+  foreach (ImageStdParams::$custom as $custom_key => $last_used)
+  {
+    $params = bratonien_tools_cache_builder_custom_params($custom_key);
+    if ($params)
+    {
+      $variants['custom:'.$custom_key] = $params;
+    }
+  }
+  return $variants;
+}
+
+function bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id)
+{
+  $status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$worker_index.'.json';
+  $variants = bratonien_tools_cache_builder_variants();
+  $rows = array();
+  $result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id');
+  while ($row = pwg_db_fetch_assoc($result))
+  {
+    if (((int)$row['id'] % $worker_count) === $worker_index)
+    {
+      $rows[] = $row;
+    }
+  }
+
+  $total = count($rows) * count($variants);
+  $completed = 0;
+  $generated = 0;
+  $cached = 0;
+  $skipped = 0;
+  $errors = 0;
+  $last_write = 0.0;
+
+  bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, 0, 0, 0, 0, 0, 'Vorbereitung');
+
+  foreach ($rows as $image_row)
+  {
+    $src = new SrcImage($image_row);
+    $source_path = $src->get_path();
+    $image_id = (int)$image_row['id'];
+    $metadata = null;
+
+    foreach ($variants as $variant_name => $requested_params)
+    {
+      $completed++;
+      $current = 'Worker '.($worker_index + 1).' · Bild #'.$image_id.' · '.$variant_name;
+
+      try
+      {
+        $derivative = new DerivativeImage($requested_params, $src);
+        $target_path = $derivative->get_path();
+        if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0)
+        {
+          $skipped++;
+        }
+        else
+        {
+          $effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path);
+          if (!$effective_params)
+          {
+            $skipped++;
+          }
+          elseif (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time)
+          {
+            $cached++;
+          }
+          elseif (!is_file($source_path) || !is_readable($source_path))
+          {
+            $errors++;
+          }
+          else
+          {
+            if ($metadata === null)
+            {
+              $metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path);
+            }
+            if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata))
+            {
+              $generated++;
+            }
+            else
+            {
+              $errors++;
+            }
+          }
+        }
+      }
+      catch (Throwable $e)
+      {
+        $errors++;
+        fwrite(STDERR, $current.': '.$e->getMessage()."\n");
+      }
+
+      $now = microtime(true);
+      if (($now - $last_write) >= 0.3 || $completed >= $total)
+      {
+        bratonien_tools_cache_builder_status($status_file, 'running', 'Worker '.($worker_index + 1).' arbeitet.', $total, $completed, $generated, $cached, $skipped, $errors, $current);
+        $last_write = $now;
+      }
+    }
+  }
+
+  $state = $errors > 0 ? 'error' : 'complete';
+  bratonien_tools_cache_builder_status(
+    $status_file,
+    $state,
+    $errors > 0 ? 'Worker '.($worker_index + 1).' mit Fehlern beendet.' : 'Worker '.($worker_index + 1).' beendet.',
+    $total, $completed, $generated, $cached, $skipped, $errors, ''
+  );
+  exit($errors > 0 ? 1 : 0);
+}
+
+function bratonien_tools_cache_builder_read_worker($file)
+{
+  if (!is_file($file) || !is_readable($file))
+  {
+    return null;
+  }
+  $raw = @file_get_contents($file);
+  $data = $raw !== false ? json_decode($raw, true) : null;
+  return is_array($data) ? $data : null;
+}
+
+function bratonien_tools_cache_builder_aggregate($run_id, $worker_count, $message)
+{
+  $sum = array('total'=>0,'completed'=>0,'generated'=>0,'cached'=>0,'skipped'=>0,'errors'=>0);
+  $current = array();
+  $states = array();
+
+  for ($i=0; $i<$worker_count; $i++)
+  {
+    $file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json';
+    $data = bratonien_tools_cache_builder_read_worker($file);
+    if (!$data)
+    {
+      continue;
+    }
+    $states[] = (string)($data['state'] ?? 'running');
+    foreach ($sum as $key => $value)
+    {
+      $sum[$key] += (int)($data[$key] ?? 0);
+    }
+    if (!empty($data['current']))
+    {
+      $current[] = $data['current'];
+    }
+  }
+
+  $state = 'running';
+  if (count($states) === $worker_count)
+  {
+    $all_done = true;
+    foreach ($states as $worker_state)
+    {
+      if ($worker_state === 'running' || $worker_state === 'queued')
+      {
+        $all_done = false;
+        break;
+      }
+    }
+    if ($all_done)
+    {
+      $state = $sum['errors'] > 0 || in_array('error', $states, true) ? 'error' : 'complete';
+    }
+  }
+
+  bratonien_tools_write_main_cache_status(array(
+    'state'=>$state,
+    'message'=>$message,
+    'total'=>$sum['total'],
+    'completed'=>$sum['completed'],
+    'generated'=>$sum['generated'],
+    'cached'=>$sum['cached'],
+    'skipped'=>$sum['skipped'],
+    'errors'=>$sum['errors'],
+    'current'=>implode(' | ', array_slice($current, 0, 4)),
+  ));
+  return $state;
+}
+
+$worker_index = null;
+$worker_count = 4;
+$run_id = '';
+foreach ($argv as $arg)
+{
+  if (preg_match('/^--worker=(\d+)$/', $arg, $m))
+  {
+    $worker_index = (int)$m[1];
+  }
+  elseif (preg_match('/^--workers=(\d+)$/', $arg, $m))
+  {
+    $worker_count = max(1, (int)$m[1]);
+  }
+  elseif (preg_match('/^--run=([A-Za-z0-9_-]+)$/', $arg, $m))
+  {
+    $run_id = $m[1];
+  }
+}
+
+if ($worker_index !== null)
+{
+  if ($run_id === '' || $worker_index < 0 || $worker_index >= $worker_count)
+  {
+    fwrite(STDERR, "Ungueltige Worker-Parameter.\n");
+    exit(1);
+  }
+  bratonien_tools_cache_builder_worker($worker_index, $worker_count, $run_id);
+}
+
 $lock_path = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.lock';
 $lock = @fopen($lock_path, 'c');
 if (!$lock || !flock($lock, LOCK_EX | LOCK_NB))
@@ -191,121 +408,92 @@ if (!$lock || !flock($lock, LOCK_EX | LOCK_NB))
   exit(0);
 }
 
-// Solange die Bratonien-Engine aktiv ist, muss Piwigos natives Wasserzeichen
-// fuer alle Derivate neutral bleiben. Der manuelle Cache-Builder erzeugt nur
-// Piwigos normalen, unveraenderten Derivatcache.
 bratonien_tools_watermark_engine_enabled();
 
-$variants = array();
+if (!function_exists('proc_open'))
-foreach (ImageStdParams::get_defined_type_map() as $type => $params)
 {
-  $variants['standard:'.$type] = $params;
+  bratonien_tools_write_main_cache_status(array('state'=>'error','message'=>'proc_open() ist deaktiviert; vier parallele Worker koennen nicht gestartet werden.','errors'=>1));
+  flock($lock, LOCK_UN);
+  fclose($lock);
+  exit(1);
 }
-foreach (ImageStdParams::$custom as $custom_key => $last_used)
+
+$run_id = date('YmdHis').'-'.getmypid();
+$log = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.log';
+$processes = array();
+
+for ($i=0; $i<$worker_count; $i++)
 {
-  $params = bratonien_tools_cache_builder_custom_params($custom_key);
+  $status_file = PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json';
-  if ($params)
+  @unlink($status_file);
+  bratonien_tools_cache_builder_status($status_file, 'queued', 'Worker '.($i + 1).' wartet.', 0, 0, 0, 0, 0, 0, '');
+
+  $command = escapeshellarg(PHP_BINARY).' '.escapeshellarg(__FILE__).' --worker='.$i.' --workers='.$worker_count.' --run='.escapeshellarg($run_id);
+  $descriptors = array(
+    0=>array('file','/dev/null','r'),
+    1=>array('file',$log,'a'),
+    2=>array('file',$log,'a'),
+  );
+  $process = @proc_open($command, $descriptors, $pipes);
+  if (!is_resource($process))
   {
-    $variants['custom:'.$custom_key] = $params;
+    bratonien_tools_cache_builder_status($status_file, 'error', 'Worker '.($i + 1).' konnte nicht gestartet werden.', 0, 0, 0, 0, 0, 1, '');
+  }
+  else
+  {
+    $processes[$i] = $process;
   }
 }
 
-$image_count = (int)pwg_db_fetch_assoc(pwg_query('SELECT COUNT(*) AS cnt FROM '.IMAGES_TABLE))['cnt'];
+bratonien_tools_write_main_cache_status(array(
-$total = $image_count * count($variants);
+  'state'=>'running',
-$completed = 0;
+  'message'=>'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.',
-$generated = 0;
+  'current'=>'Worker werden gestartet.',
-$cached = 0;
+));
-$skipped = 0;
-$errors = 0;
-$last_write = 0.0;
 
-bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, 0, 0, 0, 0, 0, 'Vorbereitung');
+while (true)
-
-$result = pwg_query('SELECT * FROM '.IMAGES_TABLE.' ORDER BY id');
-while ($image_row = pwg_db_fetch_assoc($result))
 {
-  $src = new SrcImage($image_row);
+  $running = false;
-  $source_path = $src->get_path();
+  foreach ($processes as $process)
-  $image_id = (int)$image_row['id'];
-  $metadata = null;
-
-  foreach ($variants as $variant_name => $requested_params)
   {
-    $completed++;
+    $info = proc_get_status($process);
-    $current = 'Bild #'.$image_id.' · '.$variant_name;
+    if (!empty($info['running']))
-
-    try
     {
-      $derivative = new DerivativeImage($requested_params, $src);
+      $running = true;
-      $target_path = $derivative->get_path();
-
-      // Kleine Originale koennen bei Piwigo direkt auf das Original oder auf
-      // eine bereits kleinere Standardgroesse zurueckfallen. Dann gibt es fuer
-      // diese Anforderung nichts zusaetzlich zu erzeugen.
-      if (strpos($target_path, PHPWG_ROOT_PATH.PWG_DERIVATIVE_DIR) !== 0)
-      {
-        $skipped++;
-        continue;
-      }
-
-      $effective_params = bratonien_tools_cache_builder_effective_params_from_path($target_path);
-      if (!$effective_params)
-      {
-        $skipped++;
-        continue;
-      }
-
-      if (is_file($target_path) && is_readable($target_path) && @filemtime($target_path) >= (int)$effective_params->last_mod_time)
-      {
-        $cached++;
-      }
-      else
-      {
-        if (!is_file($source_path) || !is_readable($source_path))
-        {
-          $errors++;
-        }
-        else
-        {
-          if ($metadata === null)
-          {
-            $metadata = bratonien_tools_cache_builder_metadata($image_row, $source_path);
-          }
-          if (bratonien_tools_cache_builder_generate($source_path, $target_path, $effective_params, $metadata))
-          {
-            $generated++;
-          }
-          else
-          {
-            $errors++;
-          }
-        }
-      }
-    }
-    catch (Throwable $e)
-    {
-      $errors++;
-      fwrite(STDERR, $current.': '.$e->getMessage()."\n");
     }
+  }
 
-    $now = microtime(true);
+  $state = bratonien_tools_cache_builder_aggregate($run_id, $worker_count, 'Piwigo-Bildcache wird mit 4 parallelen Workern aufgebaut.');
-    if (($now - $last_write) >= 0.3 || $completed >= $total)
+  if (!$running && ($state === 'complete' || $state === 'error'))
-    {
+  {
-      bratonien_tools_cache_builder_status('running', 'Piwigo-Bildcache wird aufgebaut.', $total, $completed, $generated, $cached, $skipped, $errors, $current);
+    break;
-      $last_write = $now;
+  }
-    }
+  usleep(500000);
+}
+
+$exit_error = false;
+foreach ($processes as $process)
+{
+  $code = proc_close($process);
+  if ($code !== 0 && $code !== -1)
+  {
+    $exit_error = true;
   }
 }
 
-$state = $errors > 0 ? 'error' : 'complete';
+$state = bratonien_tools_cache_builder_aggregate(
-$message = $errors > 0
+  $run_id,
-  ? 'Piwigo-Bildcache wurde aufgebaut, einige Varianten konnten jedoch nicht erzeugt werden.'
+  $worker_count,
-  : 'Piwigo-Bildcache wurde vollstaendig aufgebaut.';
+  $exit_error ? 'Piwigo-Bildcache beendet; mindestens ein Worker meldete einen Prozessfehler.' : 'Piwigo-Bildcache wurde aufgebaut.'
-bratonien_tools_cache_builder_status($state, $message, $total, $completed, $generated, $cached, $skipped, $errors, '');
+);
+
+for ($i=0; $i<$worker_count; $i++)
+{
+  @unlink(PHPWG_ROOT_PATH.PWG_LOCAL_DIR.'bratonien-tools-main-cache.worker-'.$run_id.'-'.$i.'.json');
+}
 
 flock($lock, LOCK_UN);
 fclose($lock);
 @unlink($lock_path);
 
-printf("Piwigo-Bildcache: %d/%d, %d neu, %d vorhanden, %d uebersprungen, %d Fehler.\n", $completed, $total, $generated, $cached, $skipped, $errors);
+exit(($state === 'error' || $exit_error) ? 1 : 0);
-exit($errors > 0 ? 1 : 0);