Pagination works incorrectly

When I use pagination then I get

Page 1

• id=10 vote=10
• id=2 vote=1
• id=3 vote=1
• id=4 vote=1
• id=5 vote=1

Page 2

• id=5 vote=1
• id=4 vote=1
• id=3 vote=1
• id=2 vote=1
• id=1 vote=1

I get the same item twice on different pages. These elements are gone

• id=6 vote=1
• id=7 vote=1
• id=8 vote=1
• id=9 vote=1

If you remove the ->orderBy('votes_count', 'desc') from the query, then the pagination works correctly, but without sorting

Page 1

• id=1 vote=1
• id=2 vote=1
• id=3 vote=1
• id=4 vote=1
• id=5 vote=1

Page 2

• id=6 vote=1
• id=7 vote=1
• id=8 vote=1
• id=9 vote=1
• id=10 vote=10

Sql Page 2

select `servers`.*, (select count(*) from `votes` where `servers`.`id` = `votes`.`server_id`) as `votes_count` from `servers` where `banned` = ? order by `votes_count` desc limit 5 offset 5

Controller

<?php

namespace App\Http\Controllers;

use Illuminate\Http\Request;

use App\Server;
use App\Tag;
use App\Version;
use App\Vote;

class ServersListHomeController extends Controller
{
    protected $server;
    protected $tag;
    protected $version;
    
    public function __construct(Server $server, Tag $tag, Version $version)
    {
        $this->server = $server;
        $this->tag = $tag;
        $this->version = $version;
    }

    public function index()
    {
        $servers = Server::where('banned', 'false')->with('version', 'tags', 'statistics')->withCount('votes')->orderBy('votes_count', 'desc')->paginate(5);
        
        return view('content.home', array(
            'servers' => $servers
        ));
    }
}