Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update CLI to use new test terminology and statuses #287

Merged
merged 5 commits into from Mar 11, 2021
Merged

Update CLI to use new test terminology and statuses #287

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7f7e844
Update CLI to use new `test` terminology and statuses
tmeasday Feb 22, 2021
0a3b7cd
Update snapshot.js
tmeasday Feb 22, 2021
bf0f4bd
Small tweak per code review
tmeasday Feb 23, 2021
ae2dcb4
Use new build statuses
tmeasday Mar 11, 2021
d4bded9
Merge branch 'next' into 286-update-apis
ghengeveld Mar 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 2 additions & 2 deletions bin/main.test.js
View file
Open in desktop
Expand Up @@ -58,7 +58,7 @@ jest.mock('node-fetch', () =>
paymentRequired: false,
},
},
snapshots: [
tests: [
{
spec: { name: 'name', component: { displayName: 'component' } },
parameters: { viewport: 320, viewportIsDefault: false },
Expand All @@ -72,7 +72,7 @@ jest.mock('node-fetch', () =>
if (query.match('TesterBuildQuery')) {
return {
data: {
app: { build: { status: 'BUILD_PENDING', changeCount: 1 } },
app: { build: { status: 'PENDING', changeCount: 1 } },
},
};
}
Expand Down
31 changes: 19 additions & 12 deletions bin/tasks/report.js
View file
Open in desktop
Expand Up @@ -16,8 +16,9 @@ const ReportQuery = `
cachedUrl
createdAt
completedAt
snapshots {
tests {
status
result
ghengeveld marked this conversation as resolved.
Show resolved Hide resolved
spec {
name
component {
Expand All @@ -43,12 +44,13 @@ export const generateReport = async (ctx) => {
const file = junitReport === true ? 'chromatic-build-{buildNumber}.xml' : junitReport;
ctx.reportPath = path.resolve(file.replace(/{buildNumber}/g, buildNumber));

const result = await client.runQuery(
const {
app: { build },
} = await client.runQuery(
ReportQuery,
{ buildNumber },
{ Authorization: `Bearer ${reportToken}` }
);
const { build } = result.app;
const buildTime = (build.completedAt || Date.now()) - build.createdAt;

const suite = reportBuilder
Expand All @@ -61,30 +63,35 @@ export const generateReport = async (ctx) => {
.property('buildUrl', build.webUrl)
.property('storybookUrl', baseStorybookUrl(build.cachedUrl));

build.snapshots.forEach(({ status, spec, parameters }) => {
build.tests.forEach(({ status, result, spec, parameters }) => {
const suffix = parameters.viewportIsDefault ? '' : ` [${parameters.viewport}px]`;
const testCase = suite
.testCase()
.className(spec.component.name.replace(/[|/]/g, '.')) // transform story path to class path
.name(`${spec.name}${suffix}`);

switch (status) {
case 'SNAPSHOT_ERROR':
case 'FAILED':
tmeasday marked this conversation as resolved.
Show resolved Hide resolved
testCase.error('Server error while taking snapshot, please try again', status);
break;
case 'SNAPSHOT_CAPTURE_ERROR':
case 'BROKEN':
testCase.error('Snapshot is broken due to an error in your Storybook', status);
break;
case 'SNAPSHOT_DENIED':
case 'DENIED':
testCase.failure('Snapshot was denied by a user', status);
break;
case 'SNAPSHOT_PENDING':
case 'PENDING':
testCase.failure('Snapshot contains visual changes and must be reviewed', status);
break;
case 'SNAPSHOT_NO_CAPTURE':
testCase.skipped();
break;
default:
default: {
switch (result) {
case 'SKIPPED':
case 'PRESERVED':
testCase.skipped();
break;
default:
}
tmeasday marked this conversation as resolved.
Show resolved Hide resolved
}
}
});

Expand Down
36 changes: 18 additions & 18 deletions bin/tasks/snapshot.js
View file
Open in desktop
Expand Up @@ -20,28 +20,28 @@ const TesterBuildQuery = `
app {
build(number: $buildNumber) {
id
status
status(legacy: false)
autoAcceptChanges
inProgressCount: snapshotCount(statuses: [SNAPSHOT_IN_PROGRESS])
snapshotCount
inProgressCount: testCount(statuses: [IN_PROGRESS])
testCount
changeCount
errorCount: snapshotCount(statuses: [SNAPSHOT_CAPTURE_ERROR])
errorCount: testCount(statuses: [BROKEN])
}
}
}
`;

export const takeSnapshots = async (ctx, task) => {
const { client, log, options } = ctx;
const { number: buildNumber, snapshots } = ctx.build;
const { number: buildNumber, tests } = ctx.build;

if (ctx.build.app.repository && ctx.uploadedBytes && !options.junitReport) {
log.info(speedUpCI(ctx.build.app.repository.provider));
}

const snapshotLabels =
const testLabels =
options.interactive &&
snapshots.map(({ spec, parameters }) => {
tests.map(({ spec, parameters }) => {
const suffix = parameters.viewportIsDefault ? '' : ` [${parameters.viewport}px]`;
return `${spec.component.displayName} › ${spec.name}${suffix}`;
});
Expand All @@ -50,14 +50,14 @@ export const takeSnapshots = async (ctx, task) => {
const { app } = await client.runQuery(TesterBuildQuery, { buildNumber });
ctx.build = { ...ctx.build, ...app.build };

if (app.build.status !== 'BUILD_IN_PROGRESS') {
if (app.build.status !== 'IN_PROGRESS') {
return ctx.build;
}

if (options.interactive) {
const { inProgressCount, snapshotCount } = ctx.build;
const cursor = snapshotCount - inProgressCount + 1;
const label = snapshotLabels[cursor - 1] || '';
const { inProgressCount, testCount } = ctx.build;
const cursor = testCount - inProgressCount + 1;
const label = testLabels[cursor - 1] || '';
task.output = pending({ ...ctx, cursor, label }).output;
}

Expand All @@ -68,16 +68,16 @@ export const takeSnapshots = async (ctx, task) => {
const build = await waitForBuild();

switch (build.status) {
case 'BUILD_PASSED':
case 'PASSED':
ctx.exitCode = 0;
ctx.log.info(buildPassedMessage(ctx));
transitionTo(buildPassed, true)(ctx, task);
break;

// They may have sneakily looked at the build while we were waiting
case 'BUILD_ACCEPTED':
case 'BUILD_PENDING':
case 'BUILD_DENIED': {
case 'ACCEPTED':
case 'PENDING':
case 'DENIED': {
if (build.autoAcceptChanges || ctx.git.matchesBranch(options.exitZeroOnChanges)) {
ctx.exitCode = 0;
ctx.log.info(buildPassedMessage(ctx));
Expand All @@ -89,14 +89,14 @@ export const takeSnapshots = async (ctx, task) => {
break;
}

case 'BUILD_FAILED':
case 'BROKEN':
ctx.exitCode = 2;
ctx.log.error(buildHasErrors(ctx));
transitionTo(buildFailed, true)(ctx, task);
break;

case 'BUILD_TIMED_OUT':
case 'BUILD_ERROR':
case 'FAILED':
case 'CANCELLED':
ctx.exitCode = 3;
transitionTo(buildError, true)(ctx, task);
break;
Expand Down
28 changes: 14 additions & 14 deletions bin/tasks/snapshot.test.js
View file
Open in desktop
Expand Up @@ -10,17 +10,17 @@ describe('takeSnapshots', () => {
const build = { app: { repository: { provider: 'github' } }, number: 1, features: {} };
const ctx = { client, env, git: { matchesBranch }, log, options: {}, build };

client.runQuery.mockReturnValueOnce({ app: { build: { status: 'BUILD_IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({ app: { build: { status: 'IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({
app: { build: { changeCount: 0, status: 'BUILD_PASSED' } },
app: { build: { changeCount: 0, status: 'PASSED' } },
});

await takeSnapshots(ctx, {});
expect(client.runQuery).toHaveBeenCalledTimes(2);
expect(client.runQuery).toHaveBeenCalledWith(expect.stringMatching(/TesterBuildQuery/), {
buildNumber: 1,
});
expect(ctx.build).toEqual({ ...build, changeCount: 0, status: 'BUILD_PASSED' });
expect(ctx.build).toEqual({ ...build, changeCount: 0, status: 'PASSED' });
expect(ctx.exitCode).toBe(0);
});

Expand All @@ -29,43 +29,43 @@ describe('takeSnapshots', () => {
const build = { app: { repository: { provider: 'github' } }, number: 1, features: {} };
const ctx = { client, env, git: { matchesBranch }, log, options: {}, build };

client.runQuery.mockReturnValueOnce({ app: { build: { status: 'BUILD_IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({ app: { build: { status: 'IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({
app: { build: { changeCount: 2, status: 'BUILD_PENDING' } },
app: { build: { changeCount: 2, status: 'PENDING' } },
});

await takeSnapshots(ctx, {});
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'BUILD_PENDING' });
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'PENDING' });
expect(ctx.exitCode).toBe(1);
});

it('sets exitCode to 2 when build fails', async () => {
it('sets exitCode to 2 when build is broken (capture error)', async () => {
const client = { runQuery: jest.fn(), setAuthorization: jest.fn() };
const build = { app: { repository: { provider: 'github' } }, number: 1, features: {} };
const ctx = { client, env, git: { matchesBranch }, log, options: {}, build };

client.runQuery.mockReturnValueOnce({ app: { build: { status: 'BUILD_IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({ app: { build: { status: 'IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({
app: { build: { changeCount: 2, status: 'BUILD_FAILED' } },
app: { build: { changeCount: 2, status: 'BROKEN' } },
});

await takeSnapshots(ctx, {});
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'BUILD_FAILED' });
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'BROKEN' });
expect(ctx.exitCode).toBe(2);
});

it('sets exitCode to 3 when build errors', async () => {
it('sets exitCode to 3 when build fails (system error)', async () => {
const client = { runQuery: jest.fn(), setAuthorization: jest.fn() };
const build = { app: { repository: { provider: 'github' } }, number: 1, features: {} };
const ctx = { client, env, git: { matchesBranch }, log, options: {}, build };

client.runQuery.mockReturnValueOnce({ app: { build: { status: 'BUILD_IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({ app: { build: { status: 'IN_PROGRESS' } } });
client.runQuery.mockReturnValueOnce({
app: { build: { changeCount: 2, status: 'BUILD_ERROR' } },
app: { build: { changeCount: 2, status: 'FAILED' } },
});

await takeSnapshots(ctx, {});
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'BUILD_ERROR' });
expect(ctx.build).toEqual({ ...build, changeCount: 2, status: 'FAILED' });
expect(ctx.exitCode).toBe(3);
});
});
6 changes: 3 additions & 3 deletions bin/tasks/verify.js
View file
Open in desktop
Expand Up @@ -12,7 +12,7 @@ const TesterCreateBuildMutation = `
id
number
specCount
snapshotCount
testCount
componentCount
webUrl
cachedUrl
Expand All @@ -33,7 +33,7 @@ const TesterCreateBuildMutation = `
}
setupUrl
}
snapshots {
tests {
spec {
name
component {
Expand Down Expand Up @@ -99,7 +99,7 @@ export const createBuild = async (ctx, task) => {
ctx.isOnboarding = build.number === 1 || (build.autoAcceptChanges && !autoAcceptChanges);

if (list) {
log.info(listingStories(build.snapshots));
log.info(listingStories(build.tests));
}

if (build.wasLimited) {
Expand Down
2 changes: 1 addition & 1 deletion bin/ui/messages/info/buildPassed.stories.js
View file
Open in desktop
Expand Up @@ -17,7 +17,7 @@ export const FirstBuildPassed = () =>
isOnboarding: true,
build: {
number: 1,
snapshotCount: 10,
testCount: 10,
componentCount: 5,
specCount: 8,
app: { setupUrl: 'https://www.chromatic.com/setup?appId=59c59bd0183bd100364e1d57' },
Expand Down
6 changes: 3 additions & 3 deletions bin/ui/tasks/snapshot.js
View file
Open in desktop
Expand Up @@ -11,7 +11,7 @@ export const initial = {
export const stats = (ctx) => ({
errors: pluralize('error', ctx.build.errorCount, true),
changes: pluralize('change', ctx.build.changeCount, true),
snapshots: pluralize('snapshot', ctx.build.snapshotCount, true),
snapshots: pluralize('snapshot', ctx.build.testCount, true),
components: pluralize('component', ctx.build.componentCount, true),
specs: pluralize('story', ctx.build.specCount, true),
});
Expand All @@ -20,8 +20,8 @@ export const pending = (ctx) => {
const { build, options, cursor = 0, label = '' } = ctx;
const { errors, snapshots, components, specs } = stats(ctx);
const only = options.only ? ` for stories matching '${options.only}'` : '';
const percentage = Math.round((cursor / build.snapshotCount) * 100);
const counts = `${cursor}/${build.snapshotCount}`;
const percentage = Math.round((cursor / build.testCount) * 100);
const counts = `${cursor}/${build.testCount}`;
const errs = build.errorCount ? `(${errors}) ` : '';
return {
status: 'pending',
Expand Down
2 changes: 1 addition & 1 deletion bin/ui/tasks/snapshot.stories.js
View file
Open in desktop
Expand Up @@ -18,7 +18,7 @@ const build = {
number: 42,
errorCount: 1,
changeCount: 2,
snapshotCount: 10,
testCount: 10,
componentCount: 5,
specCount: 8,
features: { uiTests: true },
Expand Down