Skip to content

Commit

Permalink
Provide fallback pagination when Link header unavailable (#23)
Browse files Browse the repository at this point in the history 
GitHub provides pagination through Link headers, including URLs to the
next and previous pages, as described in [0]. There was an incident
where GitHub didn't correctly return these headers for Release queries
and so the pagination for this action was broken.

Rather than relying on the Link headers to know if there are more pages
to load, we can just keep loading pages until they provide no additional
versions to parse. Using the Link headers is preferable, but this
provides an option to fall back on if the headers are not available.

[0]: https://docs.github.com/en/rest/guides/traversing-with-pagination
  • Loading branch information
@jwlawson
jwlawson committed Jan 24, 2021
1 parent 447489d commit 419c4df
Show file tree
Hide file tree
Showing 3 changed files with 180 additions and 82 deletions.
173 changes: 114 additions & 59 deletions __tests__/version.test.ts
View file
Expand Up @@ -4,7 +4,7 @@ const dataPath = path.join(__dirname, 'data');

import * as version from '../src/version';

describe('When a version is needed', () => {
describe('Pulling from multipage results with Link header', () => {
beforeEach(() => {
nock.disableNetConnect();
// Releases file contains version info for:
Expand All @@ -14,11 +14,10 @@ describe('When a version is needed', () => {
// 3.13.5
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 1 })
.replyWithFile(200, path.join(dataPath, 'releases.json'), {
'Content-Type': 'application/json',
Link:
'<...releases?page=2>; rel="next", <...releases?page=2>; rel="last"',
'<https://api.github.com/repos/Kitware/CMake/releases?page=2>; rel="next", <https://api.github.com/repos/Kitware/CMake/releases?page=2>; rel="last"',
});
// Releases file 2 contains version info for:
// 2.4.8, 2.6.4, 2.8.10.2, 2.8.12.2
Expand All @@ -30,62 +29,106 @@ describe('When a version is needed', () => {
.replyWithFile(200, path.join(dataPath, 'releases2.json'), {
'Content-Type': 'application/json',
Link:
'<...releases?page=1>; rel="prev", <...releases?page=2>; rel="last"',
'<https://api.github.com/repos/Kitware/CMake/releases?page=1>; rel="first", <https://api.github.com/repos/Kitware/CMake/releases?page=1>; rel="prev"',
});
});
afterEach(() => {
nock.cleanAll();
nock.enableNetConnect();
});
it('latest version is correctly parsed', async () => {
it('parses the latest version', async () => {
const version_info = await version.getAllVersionInfo();
const latest = await version.getLatestMatching('', version_info);
const latest = version.getLatestMatching('', version_info);
expect(latest.name).toMatch(/3.16.2/);
});
it('exact version is selected', async () => {
it('selects an exact version for full release', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('3.15.5', version_info);
const selected = version.getLatestMatching('3.15.5', version_info);
expect(selected.name).toMatch(/3.15.5/);
});
it('latest version is selected for provided minor release', async () => {
it('selects the latest version for provided minor release', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('3.15', version_info);
const selected = version.getLatestMatching('3.15', version_info);
expect(selected.name).toMatch(/3.15.6/);
});
it('latest version is selected for provided minor release with x', async () => {
it('selects the latest version for provided minor release with x', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('3.15.x', version_info);
const selected = version.getLatestMatching('3.15.x', version_info);
expect(selected.name).toMatch(/3.15.6/);
});
it('latest version is selected for provided major release with x', async () => {
it('selects the latest version for provided major release with x', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('3.x', version_info);
const selected = version.getLatestMatching('3.x', version_info);
expect(selected.name).toMatch(/3.16.2/);
});
it('non-existent full version throws', async () => {
it('throws on non-existent full version', async () => {
const version_info = await version.getAllVersionInfo();
expect(() => {
version.getLatestMatching('100.0.0', version_info);
}).toThrow('Unable to find version matching 100.0.0');
});
it('non-existent part version throws', async () => {
it('throws on non-existent part version', async () => {
const version_info = await version.getAllVersionInfo();
expect(() => {
version.getLatestMatching('100.0.x', version_info);
}).toThrow('Unable to find version matching 100.0');
});
it('versions on second page get chosen', async () => {
it('select versions on second page', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('2.x', version_info);
const selected = version.getLatestMatching('2.x', version_info);
expect(selected.name).toMatch(/2.8.12/);
});
it('extra numbers in version get ignored', async () => {
it('ignores extra numbers in version', async () => {
const version_info = await version.getAllVersionInfo();
const selected = await version.getLatestMatching('2.8.10', version_info);
const selected = version.getLatestMatching('2.8.10', version_info);
expect(selected.name).toMatch(/2.8.10/);
});
});

describe('Pulling from multipage results without Link header', () => {
// When the Link header is not available, we still want to be able to parse
// all pages. This could be done by iterating over all possible pages until
// we get no further results.
beforeEach(() => {
nock.disableNetConnect();
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.replyWithFile(200, path.join(dataPath, 'releases.json'), {
'Content-Type': 'application/json',
});
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 2 })
.replyWithFile(200, path.join(dataPath, 'releases2.json'), {
'Content-Type': 'application/json',
});
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 3 })
.reply(200, []);
});
afterEach(() => {
nock.cleanAll();
nock.enableNetConnect();
});
it('selects exact version from first page', async () => {
const version_info = await version.getAllVersionInfo();
const selected = version.getLatestMatching('3.15.5', version_info);
expect(selected.name).toMatch(/3.15.5/);
});
it('throws on non-existent full version', async () => {
const version_info = await version.getAllVersionInfo();
expect(() => {
version.getLatestMatching('100.0.0', version_info);
}).toThrow('Unable to find version matching 100.0.0');
});
it('selects versions on second page', async () => {
const version_info = await version.getAllVersionInfo();
const selected = version.getLatestMatching('2.x', version_info);
expect(selected.name).toMatch(/2.8.12/);
});
});

describe('When api token is required', () => {
beforeEach(() => {
nock('https://api.github.com', {
Expand All @@ -94,14 +137,16 @@ describe('When api token is required', () => {
},
})
.get('/repos/Kitware/CMake/releases')
.query({ page: 1 })
.replyWithFile(200, path.join(dataPath, 'releases.json'), {
'Content-Type': 'application/json',
});
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 1 })
.replyWithError('Invalid API token');
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 2 })
.reply(200, []);
});
afterEach(() => {
nock.cleanAll();
Expand All @@ -124,33 +169,38 @@ describe('When api token is required', () => {
});

describe('When using macos 3.19.2 release', () => {
const releases = {
tag_name: 'v3.19.2',
assets: [
{
name: 'cmake-3.19.2-Linux-x86_64.tar.gz',
browser_download_url:
'https://fakeaddress/cmake-3.19.2-Linux-x86_64.tar.gz',
},
{
name: 'cmake-3.19.2-macos-universal.dmg',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.2-macos-universal.dmg',
},
{
name: 'cmake-3.19.2-macos-universal.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.2-macos-universal.tar.gz',
},
],
};
const releases = [
{
tag_name: 'v3.19.2',
assets: [
{
name: 'cmake-3.19.2-Linux-x86_64.tar.gz',
browser_download_url:
'https://fakeaddress/cmake-3.19.2-Linux-x86_64.tar.gz',
},
{
name: 'cmake-3.19.2-macos-universal.dmg',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.2-macos-universal.dmg',
},
{
name: 'cmake-3.19.2-macos-universal.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.2-macos-universal.tar.gz',
},
],
},
];

beforeEach(() => {
nock.disableNetConnect();
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 1 })
.reply(200, releases);
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 2 })
.reply(200, []);
});

afterEach(() => {
Expand Down Expand Up @@ -184,28 +234,33 @@ describe('When using macos 3.19.2 release', () => {
});

describe('When providing multiple different archs', () => {
const releases = {
tag_name: 'v3.19.3',
assets: [
{
name: 'cmake-3.19.3-Linux-aarch64.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.3-Linux-aarch64.tar.gz',
},
{
name: 'cmake-3.19.3-Linux-x86_64.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.3-Linux-x86_64.tar.gz',
},
],
};
const releases = [
{
tag_name: 'v3.19.3',
assets: [
{
name: 'cmake-3.19.3-Linux-aarch64.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.3-Linux-aarch64.tar.gz',
},
{
name: 'cmake-3.19.3-Linux-x86_64.tar.gz',
browser_download_url:
'https://fakeaddress.com/cmake-3.19.3-Linux-x86_64.tar.gz',
},
],
},
];

beforeEach(() => {
nock.disableNetConnect();
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 1 })
.reply(200, releases);
nock('https://api.github.com')
.get('/repos/Kitware/CMake/releases')
.query({ page: 2 })
.reply(200, []);
});

afterEach(() => {
Expand Down
2 changes: 1 addition & 1 deletion dist/index.js
View file

Large diffs are not rendered by default.

87 changes: 65 additions & 22 deletions src/version.ts
View file
@@ -1,4 +1,5 @@
import * as rest from 'typed-rest-client/RestClient';
import * as core from '@actions/core';
import * as semver from 'semver';
import * as vi from './version-info';

Expand Down Expand Up @@ -94,44 +95,86 @@ function convertToVersionInfo(versions: GitHubVersion[]): vi.VersionInfo[] {

function getHttpOptions(
api_token: string,
page_number: number
page_number: number = 1
): rest.IRequestOptions {
let options: rest.IRequestOptions = {
queryParameters: {
params: { page: page_number },
},
};
let options: rest.IRequestOptions = {};
options.additionalHeaders = { Accept: 'application/vnd.github.v3+json' };
if (page_number > 1) {
options.queryParameters = { params: { page: page_number } };
}
if (api_token) {
options.additionalHeaders.Authorization = 'token ' + api_token;
}
return options;
}

// Parse the pagination Link header to get the next url.
// The header has the form <...url...>; rel="...", <...>; rel="..."
function getNextFromLink(link: string): string | undefined {
const rLink = /<(?<url>[A-Za-z0-9_?=.\/:-]*?)>; rel="(?<rel>\w*?)"/g;
let match;
while ((match = rLink.exec(link)) != null) {
if (match.groups && /next/.test(match.groups.rel)) {
return match.groups.url;
}
}
return;
}

export async function getAllVersionInfo(
api_token: string = ''
): Promise<vi.VersionInfo[]> {
const client = new rest.RestClient(USER_AGENT);
let cur_page = 1;
let raw_versions: GitHubVersion[] = [];
let has_next_page = true;
while (has_next_page) {
const options = getHttpOptions(api_token, cur_page);
const version_response = await client.get<GitHubVersion[]>(
VERSION_URL,
options
);
const headers: { link?: string } = version_response.headers;
if (headers.link && headers.link.match(/rel="next"/)) {
has_next_page = true;
} else {
has_next_page = false;

// Fetch the first page of releases and use that to select the pagination
// method to use for all other pages.
const options = getHttpOptions(api_token);
const version_response = await client.get<GitHubVersion[]>(
VERSION_URL,
options
);
if (version_response.statusCode != 200 || !version_response.result) {
return [];
}
let raw_versions = version_response.result;
// Try to use the Link headers for pagination. If these are not available,
// then fall back to checking all pages until an empty page of results is
// returned.
let headers: { link?: string } = version_response.headers;
if (headers.link) {
core.debug(`Using link headers for pagination`);
let next = getNextFromLink(headers.link);
while (next) {
const options = getHttpOptions(api_token);
const version_response = await client.get<GitHubVersion[]>(next, options);
if (version_response.statusCode != 200 || !version_response.result) {
break;
}
raw_versions = raw_versions.concat(version_response.result);
headers = version_response.headers;
if (!headers.link) {
break;
}
next = getNextFromLink(headers.link);
}
if (version_response.result) {
} else {
core.debug(`Using page count for pagination`);
const max_pages = 20;
let cur_page = 2;
while (cur_page <= max_pages) {
const options = getHttpOptions(api_token, cur_page);
const version_response = await client.get<GitHubVersion[]>(
VERSION_URL,
options
);
if (!version_response.result || version_response.result.length == 0) {
break;
}
raw_versions = raw_versions.concat(version_response.result);
cur_page++;
}
cur_page++;
}
core.debug(`overall got ${raw_versions.length} versions`);
const versions: vi.VersionInfo[] = convertToVersionInfo(raw_versions);
return versions;
}
Expand Down

0 comments on commit 419c4df

Please sign in to comment.