tealiedie's avatar

Need help with horizon and queues

I have an app that has a process converting uploaded videos into 3 formats. the aim is to convert these videos into wmv, webm, and x256 formats.

these processes are run in separate classes ProcessVideoX256, ProcessVideoWebm, ProcessVideoWMV and pushed into queue.

now, my problem is some of these jobs are failing due long processing which I don't exactly know how to resolve.. could be because of the server specs? or my queue and horizon configuration? I've also set the $timeout property of each jobs to 20 minutes but all are still failing after 1-2 minutes..

to give you some context. below are the details of my server and configs

Max video file upload: 200MB Server specs:

  • 1 CPU
  • 1GB RAM
  • 25GB Storage

queue config

'redis' => [
  'driver' => 'redis',
  'connection' => 'default',
  'queue' => env('REDIS_QUEUE', 'default'),
  'retry_after' => 1500,
  'block_for' => null,
  'after_commit' => false,
],

horizon config

'defaults' => [
  'supervisor-1' => [
    'connection' => 'redis',
    'queue' => explode(',', env('HORIZON_MONITORED_QUEUES', 'default,main,videoqueue')),
    'balance' => 'auto',
    'autoScalingStrategy' => 'time',
    'minProcesses' => 1,
    'maxProcesses' => 10,
    'maxTime' => 0,
    'maxJobs' => 0,
    'memory' => 128,
    'tries' => 1,
    'timeout' => 180,
    'nice' => 0,
  ],
],

'environments' => [
  'production' => [
    'supervisor-1' => [
      'maxProcesses' => 10,
      'balanceMaxShift' => 1,
      'balanceCooldown' => 3,
     ],
  ],

  'development' => [
    'supervisor-1' => [
      'maxProcesses' => 10,
      'balanceMaxShift' => 1,
      'balanceCooldown' => 3,
    ],
  ],

  'staging' => [
    'supervisor-1' => [
      'maxProcesses' => 10,
      'balanceMaxShift' => 1,
      'balanceCooldown' => 3,
    ],
  ],

  'local' => [
    'supervisor-1' => [
      'maxProcesses' => 3,
    ],
  ],
],

ProcessVideoWebm class

class ProcessVideoWebM implements ShouldQueue, ShouldBeUnique
{
	use Batchable, Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

	private $video;

	protected $timeout = 20 * 60; // 20 mins per process before timing out

	protected $tries = 3;

	protected $format;

	public $failOnTimeout = false;

	/**
	* Create a new job instance.
	*
	* @return void
	*/
	public function __construct(VideoOptimiseQueue $video)
	{
		$this->video = $video;
		$this->format = \FFMpeg\Format\Video\WebM::class;
	}

	/**
	* Execute the job.
	*
	* @return void
	*/
	public function handle()
	{
		Log::info('Process WebM');

		$processVideo = new ProcessVideo();
		$format = $this->format;

		try {
			$processVideo->optimiseVideo($this->video, $format);
		}
		catch (\Exception $e) {
			$processVideo->logErrors($this->video, $e, $format);
		}
		$processVideo->updateCodecsPending($this->video, "WebM");
	}

	/**
	 * Handle a job failure.
	 *
	 * @return void
	 */
	public function failed(Throwable $e)
	{
		$processVideo = new ProcessVideo();
		$processVideo->updateCodecsPending($this->video, "WebM");
		$processVideo->logErrors($this->video, $e, $this->format);
	}
}

main ProcessVideo

    public function optimiseVideo($video, $format)
	{
		$identifier = $video->identifier;
		
		// separate the filename and extension by the dot (it is assumed that the filename would contain only on dot)
		// eg. filename: 1615898734-il3kve5w.mp4 [time-randomstring]
		$file = explode('.', $identifier); 
		$filename = $file[0];
		$extension = $file[1];
		
		$identifierFolder = $file[0]; // identifier without the extension
		$filepath = 'queue/' . $identifierFolder. '/' . $filename . '.' . $extension;
		
		if(!Storage::disk('uploads')->exists($filepath)) {
			throw new \Illuminate\Contracts\Filesystem\FileNotFoundException($filepath);
		}

		try {
			$this->optimiseVideoForFormat($format, $filepath, $identifier, $video);
		} 
		catch (\Exception $e) {
			$this->logErrors($video, $e, $format);
		}
	}

	private function optimiseVideoForFormat($format, $filepath,  $identifier, $video)
	{
		$maxWidth  = 1600;
		$maxHeight = 1600;

		$extension = '';
		switch ($format) {
			case \FFMpeg\Format\Video\X264::class:
				$extension = '.mkv';
				break;
			case \FFMpeg\Format\Video\WMV::class:
				$extension = '.wmv';
				break;
			case \FFMpeg\Format\Video\WebM::class:
				$extension = '.webm';
				break;
		}

		$identifierWithoutExtension = substr($identifier, 0, strrpos($identifier, '.'));
		$folderPath = 'optimised/' . $identifierWithoutExtension;
		$newFilepath = $folderPath . '/' . $identifierWithoutExtension . $extension;

		try {
			$details = \ProtoneMedia\LaravelFFMpeg\Support\FFMpeg::fromDisk('uploads')
				->open($filepath)
				->getVideoStream()
				->getDimensions();

			// get dimensions to resize to
			$dimensions = $this->calculateAspectRatio($details, $maxWidth, $maxHeight);

	
			\ProtoneMedia\LaravelFFMpeg\Support\FFMpeg::fromDisk('uploads')
				->open($filepath)
				->addFilter(function (\FFMpeg\Filters\Video\VideoFilters $filters) use ($dimensions) {
					$filters->resize(new \FFMpeg\Coordinate\Dimension($dimensions[0], $dimensions[1]));
				})
				->export()
				->toDisk('uploads')
				->inFormat(new $format)
				->save($newFilepath);
		}
		catch(\FFMpeg\Exception\RuntimeException $e){
			$this->handleException($e, $format, $filepath, $video);
		}
		catch(\ProtoneMedia\LaravelFFMpeg\Exporters\EncodingException $e){
			$this->handleException($e, $format, $filepath, $video);
		}
		catch (\Exception $e) {
			$this->handleException($e, $format, $filepath, $video);
		}
	}

    private function calculateAspectRatio(\FFMpeg\Coordinate\Dimension $details, $maxWidth, $maxHeight)
	{
		$width  = $details->getWidth();
		$height = $details->getHeight();

		// get the largest downsize ratio and use that to downsize the dimensions
		$downsizeRatioW = $width / $maxWidth;
		$downsizeRatioH = $height / $maxHeight;

		// ensure that videos are not accidentally upsized
		$ratio = max(1, max($downsizeRatioW, $downsizeRatioH));

		// downsize the dimensions
		$width /= $ratio;
		$height /= $ratio;

		return [round($width), round($height)];
	}
0 likes
3 replies
marosmjartan's avatar

Maybe try adjust property timeout in the horizon config. You have timeout set to 180 seconds.

amitsolanki24_'s avatar

@tealiedie Try to update timeout 600 -> 10mint.

Please can you share your file code ProcessVideoX256 I just want to se

Please or to participate in this conversation.