From 91b148d1299b7ea1d2d14f036b9cf87652c8bca2 Mon Sep 17 00:00:00 2001
From: Royce <royce@ecs.vuw.ac.nz>
Date: Wed, 15 May 2024 11:15:44 +1200
Subject: [PATCH] Keyset pagination was added to Gitlab a while ago and as of
 Gitlab v17.0 'users' endpoint only uses keyset pagination.

This adds support for keyset pagination.

As keyset pagination returns the next page as a link in the
in the HTTP response header. RESTClient.pm was changed to return
http headers when called in list mode.

Paginator.pm checks for the pagination='keyset' parameter or 'users'
method and passes keyset parameters to GitLab API method calls.
It then parses the response header for the Link header, pulling
out the returned URL parameters. Storing them for the next call to
next_page.
---
 lib/GitLab/API/v4/Paginator.pm  | 80 +++++++++++++++++++++++++++++----
 lib/GitLab/API/v4/RESTClient.pm | 20 +++++++--
 2 files changed, 87 insertions(+), 13 deletions(-)

diff --git a/lib/GitLab/API/v4/Paginator.pm b/lib/GitLab/API/v4/Paginator.pm
index b911d6d..20d9927 100644
--- a/lib/GitLab/API/v4/Paginator.pm
+++ b/lib/GitLab/API/v4/Paginator.pm
@@ -82,6 +82,12 @@ has params => (
     default => sub{ {} },
 );
 
+has _next_params => (
+    is      => 'rw',
+    isa     => HashRef,
+    default => sub{ {} },
+);
+
 =head1 METHODS
 
 =cut
@@ -110,6 +116,9 @@ has _last_page => (
 
 Returns an array ref of records for the next page.
 
+To use keyset pagination pagination=>'keyset' must be set
+in params hash
+
 =cut
 
 sub next_page {
@@ -117,18 +126,41 @@ sub next_page {
 
     return if $self->_last_page();
 
-    my $page     = $self->_page() + 1;
+    my $page;
     my $params   = $self->params();
-    my $per_page = $params->{per_page} || 20;
+    # Don't allow cursor to be directly passed in
+    delete $params->{'cursor'};
 
-    $params = {
-        %$params,
-        page     => $page,
-        per_page => $per_page,
-    };
+    # if keyset pagination
+    my $pagination = $params->{'pagination'};
+    my $keyset = (defined $pagination && $pagination eq 'keyset') ? 1 : 0;
 
     my $method = $self->method();
-    my $records = $self->api->$method(
+    # As of Gitlab v17.0 'users'endpoint only uses keyset pagination
+    $keyset = 1 if ($method eq 'users' && ! $keyset);
+
+    my $per_page = $params->{per_page} || 20;
+
+    if ($keyset) {
+        my $next_params = $self->_next_params();
+        $params = {
+            'order_by'   => 'id',
+            'sort'       => 'asc',
+            'per_page'   => $per_page,
+            %$params,
+            'pagination' => 'keyset',
+            %$next_params,
+        };
+    } else {
+        $page     = $self->_page() + 1;
+        $params = {
+            %$params,
+            page     => $page,
+            per_page => $per_page,
+        };
+    }
+
+    my ($headers,$records) = $self->api->$method(
         @{ $self->args() },
         $params,
     );
@@ -136,7 +168,11 @@ sub next_page {
     croak("The $method method returned a non array ref value")
         if ref($records) ne 'ARRAY';
 
-    $self->_page( $page );
+    if ($keyset) {
+        $self->_next_link_params($headers)
+    } else {
+        $self->_page( $page );
+    }
     $self->_last_page( 1 ) if @$records < $per_page;
     $self->_records( [ @$records ] );
 
@@ -145,6 +181,31 @@ sub next_page {
     return $records;
 }
 
+=head2 _next_link_params
+
+  Sets _next_params hash, from the params returned in the next link,
+  when using pagination='keyset'
+
+=cut
+
+sub _next_link_params {
+    my ($self,$headers) = @_;
+    my $links = $headers->{'link'};
+    return undef if ! $links;
+    return undef if ! ($links =~ m/([^<]*)>; rel="next"/);
+    my $nextLink = $1;
+    my $params = {};
+    my ($url,$paramStr) = split('\?',$nextLink);
+    return undef if (! $paramStr);
+    my @paramList = split("&",$paramStr);
+    foreach my $param (@paramList) {
+        my ($key,$val) = split("=",$param);
+        $params->{$key} = $val;
+    }
+    $self->_next_params($params);
+    return 1;
+}
+
 =head2 next
 
     while (my $record = $paginator->next()) { ... }
@@ -207,6 +268,7 @@ sub reset {
     my ($self) = @_;
     $self->_records( [] );
     $self->_page( 0 );
+    $self->_next_params( {} );
     $self->_last_page( 0 );
     return;
 }
diff --git a/lib/GitLab/API/v4/RESTClient.pm b/lib/GitLab/API/v4/RESTClient.pm
index 62be391..319aac9 100644
--- a/lib/GitLab/API/v4/RESTClient.pm
+++ b/lib/GitLab/API/v4/RESTClient.pm
@@ -150,6 +150,9 @@ sub request {
     my $query = delete $options->{query};
     my $content = delete $options->{content};
     my $headers = $options->{headers} = { %{ $options->{headers} || {} } };
+    my $pagination = $query->{'pagination'};
+    # delete cursor and save it for later to avoid it being corrupted
+    my $cursor = delete $query->{'cursor'};
 
     # Convert foo/:bar/baz into foo/%s/baz.
     my $path = $raw_path;
@@ -163,9 +166,13 @@ sub request {
 
     my $url = $self->_clean_base_url->clone();
     $url->path( $url->path() . '/' . $path );
+    # query_form can corrupt the cursor so we add it on afterwards
     $url->query_form( $query ) if defined $query;
-    $url = "$url"; # No more changes to the url from this point forward.
-
+    $url = "$url";
+    if ($pagination && $cursor) {
+        $url = "$url&cursor=$cursor";
+    }
+    # No more changes to the url from this point forward.
     my $req_method = 'request';
     my $req = [ $verb, $url, $options ];
 
@@ -240,10 +247,15 @@ sub request {
 
         my $decode = $options->{decode};
         $decode = 1 if !defined $decode;
-        return $res->{content} if !$decode;
+
+        # returns headers,contents.
+        # When called in scalar mode, contents is returned and
+        # headers thrown away.
+        #
+        return $res->{'headers'},$res->{content} if !$decode;
 
         return try{
-            $self->json->decode( $res->{content} );
+            $res->{'headers'},$self->json->decode( $res->{content} );
         }
         catch {
             croakf(