I am dealing with a problem that needs to export all data from a table to a CSV file. Since there are plenty of rows, I divided by Model::all()->chunck(5000), and then throw the chunked collection piece into the job queue.
The error occurs when I use async queues while somehow successful when using a sync function like dispatchNow.
I am using PHP 8.07 and Laravel 6. Here is the code:
// controller
$items = $this->getCollection($data_type);
$count = $this->countItems($items);
if ($count > 10000) {
$item_collections = $items->chunk(5000);
$is_chunked = false;
foreach ($item_collections as $item_collection) {
$job = new ProcessExport($data_type, $item_collection, true, $is_chunked);
$this->dispatch($job); // lead to an error, but $this->dispatchNow($job) worked fine
$is_chunked = true;
}
// ...
}
// job
class ProcessExport implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 3;
public $timeout = 60;
private $data_type;
private $collection;
private $is_chunked;
private $is_local;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($data_type, $collection, $is_local, $is_chunked = false)
{
$this->queue = 'export';
$this->data_type = $data_type;
$this->collection = $collection;
$this->is_local = $is_local;
$this->is_chunked = $is_chunked;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
try {
$path = $this->getOutPutPath();
$data = new AllExport($this->data_type, $this->collection, $this->is_chunked);
$content = convertToSJIS($data);
$disk = $this->is_local ? 'local' : 'ftp';
Storage::disk($disk)->put($path, $content);
} catch (Exception $e) {
Log::error(__FILE__ . " (" . __LINE__ . ")" . PHP_EOL . $e->getMessage());
}
}
//...
}
// export class using Laravel Excel
use Maatwebsite\Excel\Concerns\FromCollection;
use Maatwebsite\Excel\Concerns\WithHeadings;
use App\Models\MItem;
use App\Models\MItemFactory;
use App\Models\MItemMaterial;
class AllExport implements FromCollection, WithHeadings
{
private $data_type;
private $collection;
private $is_chunked;
public function __construct($data_type, $collection, $is_chunked = false)
{
$this->data_type = $data_type;
$this->collection = $collection;
$this->is_chunked = $is_chunked;
}
/**
* @return \Illuminate\Support\Collection
*/
public function collection()
{
return $this->collection;
}
public function headings()
{
if ($this->is_chunked) {
return [];
}
switch ($this->data_type) {
//...
}
}
}
// config/queue.php, using db connection
'connections' => [
'sync' => [
'driver' => 'sync',
],
'database' => [
'driver' => 'database',
'table' => 'jobs',
'queue' => ['default', 'export'],
'retry_after' => 90,
],
//...
]
Here is the traceback, but not really understand where the problem is:
[2021-12-08 18:21:09] local.ERROR: array_key_exists(): Argument #1 ($key) must be a valid array offset type {"userId":1,"exception":"[object] (Symfony\Component\Debug\Exception\FatalThrowableError(code: 0): array_key_exists(): Argument #1 ($key) must be a valid array offset type at C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Concerns\HasAttributes.php:331)
[stacktrace]
#0 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Concerns\HasAttributes.php(331): array_key_exists(Array, Array)
#1 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Model.php(1400): Illuminate\Database\Eloquent\Model->getAttribute(Array)
#2 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Model.php(1410): Illuminate\Database\Eloquent\Model->getKey()
#3 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Support\HigherOrderCollectionProxy.php(60): Illuminate\Database\Eloquent\Model->getQueueableId()
#4 [internal function]: Illuminate\Support\HigherOrderCollectionProxy->Illuminate\Support\{closure}(Object(App\Models\MItem), 0)
#5 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Support\Collection.php(638): array_map(Object(Closure), Array, Array)
#6 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Collection.php(254): Illuminate\Support\Collection->map(Object(Closure))
#7 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Support\HigherOrderCollectionProxy.php(61): Illuminate\Database\Eloquent\Collection->map(Object(Closure))
#8 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Database\Eloquent\Collection.php(566): Illuminate\Support\HigherOrderCollectionProxy->__call('getQueueableId', Array)
#9 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\SerializesAndRestoresModelIdentifiers.php(25): Illuminate\Database\Eloquent\Collection->getQueueableIds()
#10 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\SerializesModels.php(83): App\Jobs\ProcessExport->getSerializedPropertyValue(Object(Illuminate\Database\Eloquent\Collection))
#11 [internal function]: App\Jobs\ProcessExport->__serialize()
#12 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\Queue.php(139): serialize(Object(App\Jobs\ProcessExport))
#13 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\Queue.php(110): Illuminate\Queue\Queue->createObjectPayload(Object(App\Jobs\ProcessExport), 'export')
#14 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\Queue.php(88): Illuminate\Queue\Queue->createPayloadArray(Object(App\Jobs\ProcessExport), 'export', '')
#15 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\DatabaseQueue.php(83): Illuminate\Queue\Queue->createPayload(Object(App\Jobs\ProcessExport), 'export', '')
#16 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Queue\Queue.php(44): Illuminate\Queue\DatabaseQueue->push(Object(App\Jobs\ProcessExport), '', 'export')
#17 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Bus\Dispatcher.php(175): Illuminate\Queue\Queue->pushOn('export', Object(App\Jobs\ProcessExport))
#18 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Bus\Dispatcher.php(158): Illuminate\Bus\Dispatcher->pushCommandToQueue(Object(Illuminate\Queue\DatabaseQueue), Object(App\Jobs\ProcessExport))
#19 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Bus\Dispatcher.php(73): Illuminate\Bus\Dispatcher->dispatchToQueue(Object(App\Jobs\ProcessExport))
#20 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Bus\DispatchesJobs.php(17): Illuminate\Bus\Dispatcher->dispatch(Object(App\Jobs\ProcessExport))
#21 C:\Users\den\projects\web_products\app\Http\Controllers\Maintenance\MasterDataExportController.php(63): App\Http\Controllers\Controller->dispatch(Object(App\Jobs\ProcessExport))
#22 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Controller.php(54): App\Http\Controllers\Maintenance\MasterDataExportController->export(Object(Illuminate\Http\Request))
#23 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\ControllerDispatcher.php(45): Illuminate\Routing\Controller->callAction('export', Array)
#24 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Route.php(219): Illuminate\Routing\ControllerDispatcher->dispatch(Object(Illuminate\Routing\Route), Object(App\Http\Controllers\Maintenance\MasterDataExportController), 'export')
#25 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Route.php(176): Illuminate\Routing\Route->runController()
#26 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Router.php(681): Illuminate\Routing\Route->run()
#27 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(130): Illuminate\Routing\Router->Illuminate\Routing\{closure}(Object(Illuminate\Http\Request))
#28 C:\Users\den\projects\web_products\app\Http\Middleware\PermissionMiddleware.php(23): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#29 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): App\Http\Middleware\PermissionMiddleware->handle(Object(Illuminate\Http\Request), Object(Closure), 'view-export')
#30 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Middleware\SubstituteBindings.php(41): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#31 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Routing\Middleware\SubstituteBindings->handle(Object(Illuminate\Http\Request), Object(Closure))
#32 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Auth\Middleware\Authenticate.php(43): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#33 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Auth\Middleware\Authenticate->handle(Object(Illuminate\Http\Request), Object(Closure))
#34 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Middleware\VerifyCsrfToken.php(78): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#35 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Foundation\Http\Middleware\VerifyCsrfToken->handle(Object(Illuminate\Http\Request), Object(Closure))
#36 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\View\Middleware\ShareErrorsFromSession.php(49): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#37 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\View\Middleware\ShareErrorsFromSession->handle(Object(Illuminate\Http\Request), Object(Closure))
#38 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Session\Middleware\StartSession.php(56): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#39 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Session\Middleware\StartSession->handle(Object(Illuminate\Http\Request), Object(Closure))
#40 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse.php(37): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#41 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Cookie\Middleware\AddQueuedCookiesToResponse->handle(Object(Illuminate\Http\Request), Object(Closure))
#42 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Cookie\Middleware\EncryptCookies.php(67): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#43 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Cookie\Middleware\EncryptCookies->handle(Object(Illuminate\Http\Request), Object(Closure))
#44 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(105): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#45 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Router.php(683): Illuminate\Pipeline\Pipeline->then(Object(Closure))
#46 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Router.php(658): Illuminate\Routing\Router->runRouteWithinStack(Object(Illuminate\Routing\Route), Object(Illuminate\Http\Request))
#47 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Router.php(624): Illuminate\Routing\Router->runRoute(Object(Illuminate\Http\Request), Object(Illuminate\Routing\Route))
#48 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Routing\Router.php(613): Illuminate\Routing\Router->dispatchToRoute(Object(Illuminate\Http\Request))
#49 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Kernel.php(170): Illuminate\Routing\Router->dispatch(Object(Illuminate\Http\Request))
#50 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(130): Illuminate\Foundation\Http\Kernel->Illuminate\Foundation\Http\{closure}(Object(Illuminate\Http\Request))
#51 C:\Users\den\projects\web_products\vendor\barryvdh\laravel-debugbar\src\Middleware\InjectDebugbar.php(67): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#52 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Barryvdh\Debugbar\Middleware\InjectDebugbar->handle(Object(Illuminate\Http\Request), Object(Closure))
#53 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Middleware\TransformsRequest.php(21): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#54 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Foundation\Http\Middleware\TransformsRequest->handle(Object(Illuminate\Http\Request), Object(Closure))
#55 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Middleware\TransformsRequest.php(21): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#56 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Foundation\Http\Middleware\TransformsRequest->handle(Object(Illuminate\Http\Request), Object(Closure))
#57 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Middleware\ValidatePostSize.php(27): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#58 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Foundation\Http\Middleware\ValidatePostSize->handle(Object(Illuminate\Http\Request), Object(Closure))
#59 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode.php(63): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#60 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Illuminate\Foundation\Http\Middleware\CheckForMaintenanceMode->handle(Object(Illuminate\Http\Request), Object(Closure))
#61 C:\Users\den\projects\web_products\vendor\fideloper\proxy\src\TrustProxies.php(57): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#62 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(171): Fideloper\Proxy\TrustProxies->handle(Object(Illuminate\Http\Request), Object(Closure))
#63 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Pipeline\Pipeline.php(105): Illuminate\Pipeline\Pipeline->Illuminate\Pipeline\{closure}(Object(Illuminate\Http\Request))
#64 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Kernel.php(145): Illuminate\Pipeline\Pipeline->then(Object(Closure))
#65 C:\Users\den\projects\web_products\vendor\laravel\framework\src\Illuminate\Foundation\Http\Kernel.php(110): Illuminate\Foundation\Http\Kernel->sendRequestThroughRouter(Object(Illuminate\Http\Request))
#66 C:\Users\den\projects\web_products\public\index.php(55): Illuminate\Foundation\Http\Kernel->handle(Object(Illuminate\Http\Request))
#67 C:\Users\den\projects\web_products\server.php(21): require_once('C:\\Users\\den\\pr...')
#68 {main}
"}
Does anyone have any clues on this issue? Really appreciate it.
