Skip to content

Commit

Permalink
fix(operations): fail fast the current batch to respect the operation…
Browse files Browse the repository at this point in the history
…s limit (#474)

* fix(operations): fail fast the current batch to respect the operations limit

Instead of processing an entire batch of 100 issues before checking the operations left, simply do it before processing an issue so that we respect as expected the limitation of the operations per run
Fixes #466

* test(debug): disable the dry-run for the test by default

we will be able to test the operations per run and have more complete logs that could help us debug the workflow

* chore(logs): also display the stats when the operations per run stopped the workflow

* chore(stats): fix a bad stats related to the consumed operations

* test(operations-per-run): add coverage

* chore: update index
  • Loading branch information
C0ZEN committed Jun 7, 2021
1 parent 8deaf75 commit 5f6f311
Show file tree
Hide file tree
Showing 5 changed files with 267 additions and 24 deletions.
2 changes: 1 addition & 1 deletion __tests__/constants/default-processor-options.ts
Expand Up @@ -25,7 +25,7 @@ export const DefaultProcessorOptions: IIssuesProcessorOptions = Object.freeze({
anyOfIssueLabels: '',
anyOfPrLabels: '',
operationsPerRun: 100,
debugOnly: true,
debugOnly: false,
removeStaleWhenUpdated: false,
removeIssueStaleWhenUpdated: undefined,
removePrStaleWhenUpdated: undefined,
Expand Down
230 changes: 230 additions & 0 deletions __tests__/operations-per-run.spec.ts
@@ -0,0 +1,230 @@
import {Issue} from '../src/classes/issue';
import {IIssuesProcessorOptions} from '../src/interfaces/issues-processor-options';
import {IsoDateString} from '../src/types/iso-date-string';
import {IssuesProcessorMock} from './classes/issues-processor-mock';
import {DefaultProcessorOptions} from './constants/default-processor-options';
import {generateIssue} from './functions/generate-issue';

describe('operations per run option', (): void => {
let sut: SUT;

beforeEach((): void => {
sut = new SUT();
});

describe('when one issue should be stale within 10 days and updated 20 days ago', (): void => {
beforeEach((): void => {
sut.staleIn(10).newIssue().updated(20);
});

describe('when the operations per run option is set to 1', (): void => {
beforeEach((): void => {
sut.operationsPerRun(1);
});

it('should consume 1 operation (stale label)', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(1);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(1);
});
});
});

describe('when one issue should be stale within 10 days and updated 20 days ago and a comment should be added when stale', (): void => {
beforeEach((): void => {
sut.staleIn(10).commentOnStale().newIssue().updated(20);
});

describe('when the operations per run option is set to 2', (): void => {
beforeEach((): void => {
sut.operationsPerRun(2);
});

it('should consume 2 operations (stale label, comment)', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(1);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(2);
});
});

// Special case were we continue the issue processing even if the operations per run is reached
describe('when the operations per run option is set to 1', (): void => {
beforeEach((): void => {
sut.operationsPerRun(1);
});

it('should consume 2 operations (stale label, comment)', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(1);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(2);
});
});
});

describe('when two issues should be stale within 10 days and updated 20 days ago and a comment should be added when stale', (): void => {
beforeEach((): void => {
sut.staleIn(10).commentOnStale();
sut.newIssue().updated(20);
sut.newIssue().updated(20);
});

describe('when the operations per run option is set to 3', (): void => {
beforeEach((): void => {
sut.operationsPerRun(3);
});

it('should consume 4 operations (stale label, comment)', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(2);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(4);
});
});

describe('when the operations per run option is set to 2', (): void => {
beforeEach((): void => {
sut.operationsPerRun(2);
});

it('should consume 2 operations (stale label, comment) and stop', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(1);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(2);
});
});

// Special case were we continue the issue processing even if the operations per run is reached
describe('when the operations per run option is set to 1', (): void => {
beforeEach((): void => {
sut.operationsPerRun(1);
});

it('should consume 2 operations (stale label, comment) and stop', async () => {
expect.assertions(2);

await sut.test();

expect(sut.processor.staleIssues).toHaveLength(1);
expect(
sut.processor.operations.getConsumedOperationsCount()
).toStrictEqual(2);
});
});
});
});

class SUT {
processor!: IssuesProcessorMock;
private _opts: IIssuesProcessorOptions = {
...DefaultProcessorOptions,
staleIssueMessage: ''
};
private _testIssueList: Issue[] = [];
private _sutIssues: SUTIssue[] = [];

newIssue(): SUTIssue {
const sutIssue: SUTIssue = new SUTIssue();
this._sutIssues.push(sutIssue);

return sutIssue;
}

staleIn(days: number): SUT {
this._updateOptions({
daysBeforeIssueStale: days
});

return this;
}

commentOnStale(): SUT {
this._updateOptions({
staleIssueMessage: 'Dummy stale issue message'
});

return this;
}

operationsPerRun(count: number): SUT {
this._updateOptions({
operationsPerRun: count
});

return this;
}

async test(): Promise<number> {
return this._setTestIssueList()._setProcessor();
}

private _updateOptions(opts: Partial<IIssuesProcessorOptions>): SUT {
this._opts = {...this._opts, ...opts};

return this;
}

private _setTestIssueList(): SUT {
this._testIssueList = this._sutIssues.map(
(sutIssue: SUTIssue): Issue => {
return generateIssue(
this._opts,
1,
'My first issue',
sutIssue.updatedAt,
sutIssue.updatedAt,
false
);
}
);

return this;
}

private async _setProcessor(): Promise<number> {
this.processor = new IssuesProcessorMock(
this._opts,
async () => 'abot',
async p => (p === 1 ? this._testIssueList : []),
async () => [],
async () => new Date().toDateString()
);

return this.processor.processIssues(1);
}
}

class SUTIssue {
updatedAt: IsoDateString = '2020-01-01T17:00:00Z';

updated(daysAgo: number): SUTIssue {
const today = new Date();
today.setDate(today.getDate() - daysAgo);
this.updatedAt = today.toISOString();

return this;
}
}
29 changes: 17 additions & 12 deletions dist/index.js
Expand Up @@ -255,7 +255,7 @@ class IssuesProcessor {
this.removedLabelIssues = [];
this.options = options;
this.client = github_1.getOctokit(this.options.repoToken);
this._operations = new stale_operations_1.StaleOperations(this.options);
this.operations = new stale_operations_1.StaleOperations(this.options);
this._logger.info(logger_service_1.LoggerService.yellow(`Starting the stale action process...`));
if (this.options.debugOnly) {
this._logger.warning(logger_service_1.LoggerService.yellowBright(`Executing in debug mode!`));
Expand Down Expand Up @@ -286,20 +286,24 @@ class IssuesProcessor {
return issue.isPullRequest ? option_1.Option.ClosePrLabel : option_1.Option.CloseIssueLabel;
}
processIssues(page = 1) {
var _a, _b;
var _a, _b, _c;
return __awaiter(this, void 0, void 0, function* () {
// get the next batch of issues
const issues = yield this.getIssues(page);
const actor = yield this.getActor();
if (issues.length <= 0) {
this._logger.info(logger_service_1.LoggerService.green(`No more issues found to process. Exiting...`));
(_a = this._statistics) === null || _a === void 0 ? void 0 : _a.setRemainingOperations(this._operations.getRemainingOperationsCount()).logStats();
return this._operations.getRemainingOperationsCount();
(_a = this._statistics) === null || _a === void 0 ? void 0 : _a.setOperationsCount(this.operations.getConsumedOperationsCount()).logStats();
return this.operations.getRemainingOperationsCount();
}
else {
this._logger.info(`${logger_service_1.LoggerService.yellow('Processing the batch of issues')} ${logger_service_1.LoggerService.cyan(`#${page}`)} ${logger_service_1.LoggerService.yellow('containing')} ${logger_service_1.LoggerService.cyan(issues.length)} ${logger_service_1.LoggerService.yellow(`issue${issues.length > 1 ? 's' : ''}...`)}`);
}
for (const issue of issues.values()) {
// Stop the processing if no more operations remains
if (!this.operations.hasRemainingOperations()) {
break;
}
const issueLogger = new issue_logger_1.IssueLogger(issue);
(_b = this._statistics) === null || _b === void 0 ? void 0 : _b.incrementProcessedItemsCount(issue);
issueLogger.info(`Found this $$type last updated at: ${logger_service_1.LoggerService.cyan(issue.updated_at)}`);
Expand Down Expand Up @@ -450,9 +454,10 @@ class IssuesProcessor {
}
IssuesProcessor._endIssueProcessing(issue);
}
if (!this._operations.hasRemainingOperations()) {
if (!this.operations.hasRemainingOperations()) {
this._logger.warning(logger_service_1.LoggerService.yellowBright(`No more operations left! Exiting...`));
this._logger.warning(`${logger_service_1.LoggerService.yellowBright('If you think that not enough issues were processed you could try to increase the quantity related to the')} ${this._logger.createOptionLink(option_1.Option.OperationsPerRun)} ${logger_service_1.LoggerService.yellowBright('option which is currently set to')} ${logger_service_1.LoggerService.cyan(this.options.operationsPerRun)}`);
(_c = this._statistics) === null || _c === void 0 ? void 0 : _c.setOperationsCount(this.operations.getConsumedOperationsCount()).logStats();
return 0;
}
this._logger.info(`${logger_service_1.LoggerService.green('Batch')} ${logger_service_1.LoggerService.cyan(`#${page}`)} ${logger_service_1.LoggerService.green('processed.')}`);
Expand All @@ -466,7 +471,7 @@ class IssuesProcessor {
return __awaiter(this, void 0, void 0, function* () {
// Find any comments since date on the given issue
try {
this._operations.consumeOperation();
this.operations.consumeOperation();
(_a = this._statistics) === null || _a === void 0 ? void 0 : _a.incrementFetchedItemsCommentsCount();
const comments = yield this.client.issues.listComments({
owner: github_1.context.repo.owner,
Expand All @@ -487,7 +492,7 @@ class IssuesProcessor {
return __awaiter(this, void 0, void 0, function* () {
let actor;
try {
this._operations.consumeOperation();
this.operations.consumeOperation();
actor = yield this.client.users.getAuthenticated();
}
catch (error) {
Expand All @@ -503,7 +508,7 @@ class IssuesProcessor {
// generate type for response
const endpoint = this.client.issues.listForRepo;
try {
this._operations.consumeOperation();
this.operations.consumeOperation();
const issueResult = yield this.client.issues.listForRepo({
owner: github_1.context.repo.owner,
repo: github_1.context.repo.repo,
Expand Down Expand Up @@ -871,7 +876,7 @@ class IssuesProcessor {
});
}
_consumeIssueOperation(issue) {
this._operations.consumeOperation();
this.operations.consumeOperation();
issue.operations.consumeOperation();
}
_getDaysBeforeStaleUsedOptionName(issue) {
Expand Down Expand Up @@ -1283,8 +1288,8 @@ class Statistics {
}
return this._incrementUndoStaleIssuesCount(increment);
}
setRemainingOperations(remainingOperations) {
this._operationsCount = remainingOperations;
setOperationsCount(operationsCount) {
this._operationsCount = operationsCount;
return this;
}
incrementClosedItemsCount(issue, increment = 1) {
Expand Down Expand Up @@ -8892,4 +8897,4 @@ module.exports = require("zlib");;
/******/ // Load entry module and return exports
/******/ return __nccwpck_require__(3109);
/******/ })()
;
;

0 comments on commit 5f6f311

Please sign in to comment.