Jobs and Queues - Best Practice
So I'm currently making an application which is downloading information from different API resources.
I have an output model which has a DOI (digital object identifier), and an abstract.
The majority of Output's information is derived from one API. I fire off a job and it collects all the information correctly.
I want to fire off a second job to get the Abstract from a different API.
The abstracts are in the Eutilz Pubmed Service and I've currently got the information through guzzle requests. This works fine for single jobs.
class ProcessAbstract implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 120;
protected $output;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Output $output)
{
$this->onQueue('abstracts');
$this->onConnection('redis');
$this->output= $output;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
Redis::funnel('key')->limit(1)->then(function () {
// Job logic...
$abstract = new AbstractFetcher($this->output);
$abstract->fetch();
}, function () {
// Could not obtain lock...
return $this->release(10);
});
}
}
class AbstractFetcher
{
protected $output;
public $apikey ='*************';
public function __construct(Output $output)
{
$this->doi = $output->doi;
$this->output = $output;
}
public function fetch(){
//TODO check the response codes
try{
$client = LaravelGuzzleThrottle::client(['base_uri' => 'https://eutils.ncbi.nlm.nih.gov/']);
$res = $client->get($this->getAPIUrl());
Storage::append('Fuzzle.log', $res->getStatusCode());
$xmlid = simplexml_load_string($res->getBody());
$xmlid = $xmlid->IdList->Id->__toString();
$client = LaravelGuzzleThrottle::client(['base_uri' => 'https://eutils.ncbi.nlm.nih.gov/']);
$res = $client->get($this->getOutputUrl($xmlid));
$xmlid = simplexml_load_string($res->getBody());
if(isset($xmlid->PubmedArticle->MedlineCitation->Article->Abstract->AbstractText)){
$stringpieces=[];
foreach($xmlid->PubmedArticle->MedlineCitation->Article->Abstract->AbstractText as $text) {
$stringpieces[]= (string) $text;
}
$abstract= implode(' ', $stringpieces);
Output::where('doi', $this->doi)->update(['abstract' => $abstract]);
} else {
die();
}
} catch(Exception $e){
Log::alert('Error with processing abstract');
Log::alert('Output ID ' . $this->output->id);
Log::alert('Error Message' . $e->getMessage());
Log::alert('API Url ' . $this->getAPIUrl());
Log::alert('Output Url ' . $this->getOutputUrl('test'));
};
}
public function getAPIUrl (){
return 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&WebEnv=1&usehistory=y&term='.$this->doi . '$api_key=' . $this->apikey;
}
public function getOutputUrl($xmid){
$field = array('db' => 'pubmed', 'retmode' => 'xml', 'rettype' => 'abstract', 'id' => $xmid);
return "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?".http_build_query($field);
}
}
It also correctly fetches all empty abstracts I for-loop through a collection of Outputs with empty abstracts as a single job.
class ProcessEmptyAbstracts implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct()
{
$this->onQueue('abstracts');
$this->onConnection('redis');
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
Redis::throttle('key')->allow(1)->every(5)->then(function () {
$outputs = Output::where('abstract', null)->get();
foreach ($outputs as $item) {
$url = $this->getAPIUrl($item->doi);
$xml_str = file_get_contents($url); //grab the contents
$xml = new SimpleXMLElement($xml_str); //convert to SimpleXML
// removes all the actual ones without any abstracts, content pages etc
if (!empty($xml->ErrorList)) {
$outputs->forget($item->id);
} // hasnt thrown an error so why is it not putting the abstract?
else {
$xmlid = $xml->xpath('IdList/Id');
$xmlid = strval($xmlid[0]);
$curl = curl_init();
$field = array('db' => 'pubmed', 'retmode' => 'text', 'rettype' => 'abstract', 'id' => $xmlid);
curl_setopt_array($curl, array(
CURLOPT_URL => "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?".http_build_query($field),
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_TIMEOUT => 30000,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "GET",
CURLINFO_HEADER_OUT => true,
CURLOPT_HTTPHEADER => array(
'Content-Type: application/json',
),
));
$response = curl_exec($curl);
$err = curl_error($curl);
Output::where('doi', $item->doi)->update(['abstract' => $response]);
}
sleep(1);
}
}, function () {
return $this->release(10);
});
//
}
public function getAPIUrl ($doi){
return 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&WebEnv=1&usehistory=y&term='.$doi;
}
}
Now if I try to fire off the job through a model observer, it returns multiple errors of
[2019-05-12 14:21:02] local.ERROR: App\Jobs\ProcessAbstract has been attempted too many times or run too long. The job may have previously timed out. {"exception":"[object] (Illuminate\Queue\MaxAttemptsExceededException(code: 0): App\Jobs\ProcessAbstract has been attempted too many times or run too long. The job may have previously timed out. at /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Worker.php:405) [stacktrace] #0 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(321): Illuminate\Queue\Worker->markJobAsFailedIfAlreadyExceedsMaxAttempts('redis', Object(Illuminate\Queue\Jobs\RedisJob), 1) #1 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(277): Illuminate\Queue\Worker->process('redis', Object(Illuminate\Queue\Jobs\RedisJob), Object(Illuminate\Queue\WorkerOptions)) #2 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Worker.php(118): Illuminate\Queue\Worker->runJob(Object(Illuminate\Queue\Jobs\RedisJob), 'redis', Object(Illuminate\Queue\WorkerOptions)) #3 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(102): Illuminate\Queue\Worker->daemon('redis', 'abstracts', Object(Illuminate\Queue\WorkerOptions)) #4 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Queue/Console/WorkCommand.php(86): Illuminate\Queue\Console\WorkCommand->runWorker('redis', 'abstracts') #5 /home/vagrant/code/homestead/vendor/laravel/horizon/src/Console/WorkCommand.php(46): Illuminate\Queue\Console\WorkCommand->handle() #6 [internal function]: Laravel\Horizon\Console\WorkCommand->handle() #7 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(32): call_user_func_array(Array, Array) #8 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(90): Illuminate\Container\BoundMethod::Illuminate\Container\{closure}() #9 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Container/BoundMethod.php(34): Illuminate\Container\BoundMethod::callBoundMethod(Object(Illuminate\Foundation\Application), Array, Object(Closure)) #10 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Container/Container.php(576): Illuminate\Container\BoundMethod::call(Object(Illuminate\Foundation\Application), Array, Array, NULL) #11 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Console/Command.php(183): Illuminate\Container\Container->call(Array) #12 /home/vagrant/code/homestead/vendor/symfony/console/Command/Command.php(255): Illuminate\Console\Command->execute(Object(Symfony\Component\Console\Input\ArgvInput), Object(Illuminate\Console\OutputStyle)) #13 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Console/Command.php(170): Symfony\Component\Console\Command\Command->run(Object(Symfony\Component\Console\Input\ArgvInput), Object(Illuminate\Console\OutputStyle)) #14 /home/vagrant/code/homestead/vendor/symfony/console/Application.php(908): Illuminate\Console\Command->run(Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #15 /home/vagrant/code/homestead/vendor/symfony/console/Application.php(269): Symfony\Component\Console\Application->doRunCommand(Object(Laravel\Horizon\Console\WorkCommand), Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #16 /home/vagrant/code/homestead/vendor/symfony/console/Application.php(145): Symfony\Component\Console\Application->doRun(Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #17 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Console/Application.php(90): Symfony\Component\Console\Application->run(Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #18 /home/vagrant/code/homestead/vendor/laravel/framework/src/Illuminate/Foundation/Console/Kernel.php(122): Illuminate\Console\Application->run(Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #19 /home/vagrant/code/homestead/artisan(37): Illuminate\Foundation\Console\Kernel->handle(Object(Symfony\Component\Console\Input\ArgvInput), Object(Symfony\Component\Console\Output\ConsoleOutput)) #20 {main} "}
I'm really sorry if this is obvious, but I have very limited PHP experience. I can't seem to understand why it intermittently fails one way but not another?
Please or to participate in this conversation.