set up end-to-end testing

This commit is contained in:
melody-universe 2021-08-20 16:51:35 -07:00
parent 96aaf80c13
commit 08342d1399
10 changed files with 4714 additions and 2623 deletions

20
.github/workflows/pull-request.yml vendored Normal file
View File

@ -0,0 +1,20 @@
name: Confirm Pull Request
on:
pull_request:
concurrency: end-to-end-test
jobs:
end-to-end-tests:
name: Conduct end-to-end tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- run: npm ci
name: Install dependencies
- run: npm run end-to-end-test
env:
TEST_REPO: ${{ secrets.TEST_REPO }}
TEST_USER: ${{ secrets.TEST_USER }}
TEST_TOKEN: ${{ secrets.TEST_TOKEN }}

2
.gitignore vendored
View File

@ -1,2 +1,4 @@
node_modules node_modules
.nyc_output .nyc_output
/test-repo
/.env

6849
package-lock.json generated

File diff suppressed because it is too large Load Diff

View File

@ -16,15 +16,22 @@
"main": "index.js", "main": "index.js",
"scripts": { "scripts": {
"start": "node ./index.js", "start": "node ./index.js",
"test": "jest" "test": "npm run end-to-end-test",
"end-to-end-test": "jest --roots=tests/end-to-end --testTimeout=300000 --runInBand"
}, },
"dependencies": { "dependencies": {
"actions-toolkit": "^2.2.0", "actions-toolkit": "^2.2.0",
"yargs": "^15.3.1" "yargs": "^15.3.1"
}, },
"devDependencies": { "devDependencies": {
"jest": "^25.2.7", "@types/jest": "^27.0.1",
"dotenv": "^10.0.0",
"execa": "^5.1.1",
"jest": "^25.5.4",
"js-yaml": "^4.1.0",
"node-fetch": "^2.6.1",
"prettier": "^2.3.0",
"standard": "^14.3.3", "standard": "^14.3.3",
"prettier": "^2.3.0" "tiny-glob": "^0.2.9"
} }
} }

View File

@ -0,0 +1,85 @@
const { default: fetch } = require('node-fetch');
async function clearWorkflowRuns() {
const runs = await getWorkflowRuns();
const basePath = getActionsBasePath();
await Promise.all(runs.map((run) => api(`${basePath}/runs/${run.id}`, { method: 'DELETE' })));
}
exports.clearWorkflowRuns = clearWorkflowRuns;
async function getMostRecentWorkflowRun() {
const runs = await getWorkflowRuns();
if (runs.length === 0) {
return null;
}
const mostRecentRun = runs.reduce((previous, current) => {
const prevDate = new Date(previous.created_at);
const currDate = new Date(current.created_at);
if (prevDate < currDate) {
return current;
} else {
return previous;
}
});
return mostRecentRun;
}
exports.getMostRecentWorkflowRun = getMostRecentWorkflowRun;
async function getWorkflowRun(id) {
const basePath = getActionsBasePath();
const run = await api(`${basePath}/runs/${id}`);
return run;
}
exports.getWorkflowRun = getWorkflowRun;
async function getWorkflowRuns() {
const basePath = getActionsBasePath();
const result = await api(`${basePath}/runs`);
return result.workflow_runs || [];
}
function getActionsBasePath() {
const repoUrl = process.env.TEST_REPO;
const match = /\/([^/]*)\/([^/]*)\.git$/.exec(repoUrl);
const owner = match[1];
const repo = match[2];
return `repos/${owner}/${repo}/actions`;
}
const retryAttempts = 10;
const retryInterval = 10;
async function api(path, options) {
options = options || {};
const username = process.env.TEST_USER;
const token = process.env.TEST_TOKEN;
for (let attempts = 0; attempts < retryAttempts; attempts++) {
const response = await fetch(`https://api.github.com/${path}`, {
method: options.method || 'GET',
headers: {
Authorization: `Basic ${Buffer.from(`${username}:${token}`, 'ascii').toString('base64')}`,
accept: 'application/vnd.github.v3+json',
'User-Agent': username,
...options.headers,
},
});
if (response.ok) {
const responseText = await response.text();
if (responseText.length > 0) {
return JSON.parse(responseText);
} else {
return;
}
} else if (response.status === 404) {
console.log(
`Received a 404 error while executing request at ${path}. Waiting ${retryInterval} seconds and retrying...`,
);
await new Promise((resolve) => setTimeout(resolve, retryInterval * 1000));
} else {
throw new Error(`${response.status} ${response.statusText}`);
}
}
throw new Error(
`Received ${retryAttempts} 404 errors in a row while executing request at ${path} in ${retryInterval}-second intervals.`,
);
}

View File

@ -0,0 +1,140 @@
suites:
- name: default
yaml:
name: Bump Version
on:
push:
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: version-bump
uses: ./action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
tests:
- message: no keywords
expected:
version: 1.0.1
- message: feat
expected:
version: 1.1.0
- message: minor
expected:
version: 1.2.0
- message: BREAKING CHANGE
expected:
version: 2.0.0
- message: major
expected:
version: 3.0.0
- message: pre-alpha
expected:
version: 3.0.1-alpha.0
- message: pre-beta
expected:
version: 3.0.1-beta.0
- message: pre-rc
expected:
version: 3.0.1-rc.0
- name: no-push
yaml:
name: Do Nothing
on:
push:
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: version-bump
uses: ./action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
push: false
tests:
- message: no keywords
expected:
version: 3.0.1-rc.0
- name: custom-wording
yaml:
name: Bump Version (Custom Wording)
on:
push:
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: version-bump
uses: ./action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
minor-wording: custom-minor
major-wording: custom-major
rc-wording: custom-pre
tests:
- message: custom-minor
expected:
version: 3.1.0
- message: custom-major
expected:
version: 4.0.0
- message: custom-pre
expected:
version: 4.0.1-pre.0
- name: null-default
yaml:
name: Bump Version (Default="Minor")
on:
push:
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: version-bump
uses: ./action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
default: minor
patch-wording: patch
tests:
- message: no hint
expected:
version: 4.1.0
- message: patch
expected:
version: 4.1.1
- name: custom-tag-prefix
yaml:
name: Bump Version (Custom Tag Prefix)
on:
push:
jobs:
bump-version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- id: version-bump
uses: ./action
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag-prefix: 'v'
tests:
- message: no keywords
expected:
version: 4.1.2
tag: v4.1.2
actionFiles:
- index.js
- Dockerfile
- package.json
- package-lock.json
- action.yml
image: catthehacker/ubuntu:act-latest

17
tests/end-to-end/exec.js Normal file
View File

@ -0,0 +1,17 @@
const execa = require('execa');
module.exports = async function exec(command, options, ...params) {
let suppressOutput;
if (typeof options === 'object') {
suppressOutput = options.suppressOutput;
} else {
params.unshift(options);
suppressOutput = false;
}
const subprocess = execa(command, params);
if (!suppressOutput) {
subprocess.stdout.pipe(process.stdout);
}
subprocess.stderr.pipe(process.stderr);
return await subprocess;
};

20
tests/end-to-end/git.js Normal file
View File

@ -0,0 +1,20 @@
const exec = require('./exec');
function git(options, ...params) {
return exec('git', options, ...params);
}
let firstCommit = true;
function push() {
if (firstCommit) {
firstCommit = false;
return git('push', '--force', '--set-upstream', 'origin', 'main');
} else {
return git('push');
}
}
module.exports = {
push,
default: git,
};

View File

@ -0,0 +1,122 @@
const dotenv = require('dotenv');
const setupTestRepo = require('./setupTestRepo');
const yaml = require('js-yaml');
const { readFileSync } = require('fs');
const { writeFile, readFile, mkdir } = require('fs/promises');
const { resolve, join } = require('path');
const { cwd } = require('process');
const { default: git, push: gitPush } = require('./git');
const { getMostRecentWorkflowRun, getWorkflowRun } = require('./actionsApi');
dotenv.config();
const config = getTestConfig();
beforeAll(() => setupTestRepo(config.actionFiles));
config.suites.forEach((suite) => {
const suiteYaml = yaml.dump(suite.yaml);
describe(suite.name, () => {
beforeAll(async () => {
const pushYamlPath = join('.github', 'workflows', 'push.yml');
await mkdir(join(cwd(), '.github', 'workflows'), { recursive: true });
await writeFile(join(cwd(), pushYamlPath), suiteYaml);
await git('add', pushYamlPath);
});
suite.tests.forEach((commit) => {
test(commit.message, async () => {
await generateReadMe(commit, suiteYaml);
await git('commit', '--message', commit.message);
const mostRecentDate = await getMostRecentWorkflowRunDate();
await gitPush();
const completedRun = await getCompletedRunAfter(mostRecentDate);
expect(completedRun.conclusion).toBe('success');
await git('pull');
await assertExpectation(commit.expected);
});
});
});
});
function getTestConfig() {
const path = resolve(__dirname, './config.yaml');
const buffer = readFileSync(path);
const contents = buffer.toString();
const config = yaml.load(contents);
return config;
}
async function generateReadMe(commit, suiteYaml) {
const readmePath = 'README.md';
const readMeContents = [
'# Test Details',
'## .github/workflows/push.yml',
'```YAML',
yaml.dump(suiteYaml),
'```',
'## Message',
commit.message,
'## Expectation',
`**Version:** ${commit.expected.version}`,
].join('\n');
await writeFile(join(cwd(), readmePath), readMeContents);
await git('add', readmePath);
}
async function getCompletedRunAfter(date) {
const run = await pollFor(getMostRecentWorkflowRun, (run) => run !== null && new Date(run.created_at) > date);
const completedRun = await pollFor(
() => getWorkflowRun(run.id),
(run) => run.status === 'completed',
);
return completedRun;
}
function pollFor(getResult, validateResult) {
return new Promise((resolve, reject) => {
pollAndRetry();
async function pollAndRetry() {
try {
const result = await getResult();
if (validateResult(result)) {
resolve(result);
} else {
setTimeout(pollAndRetry, 1000);
}
} catch (error) {
reject(error);
}
}
});
}
async function getMostRecentWorkflowRunDate() {
const run = await getMostRecentWorkflowRun();
const date = run === null ? new Date(0) : new Date(run.created_at);
return date;
}
async function assertExpectation({ version: expectedVersion, tag: expectedTag }) {
if (expectedTag === undefined) {
expectedTag = expectedVersion;
}
const [packageVersion, latestTag] = await Promise.all([getPackageJsonVersion(), getLatestTag()]);
expect(packageVersion).toBe(expectedVersion);
expect(latestTag).toBe(expectedTag);
}
async function getPackageJsonVersion() {
const path = join(cwd(), 'package.json');
const contents = await readFile(path);
const json = JSON.parse(contents);
return json.version;
}
async function getLatestTag() {
const result = await git({ suppressOutput: true }, 'describe', '--tags', '--abbrev=0');
return result.stdout;
}

View File

@ -0,0 +1,69 @@
const { existsSync } = require('fs');
const { rm, mkdir, copyFile, stat } = require('fs/promises');
const { chdir, cwd } = require('process');
const { resolve, join, dirname } = require('path');
const exec = require('./exec');
const { default: git } = require('./git');
const glob = require('tiny-glob');
const { clearWorkflowRuns } = require('./actionsApi');
module.exports = async function setupTestRepo(actionFileGlobPaths) {
const testRepoPath = resolve(__dirname, '..', '..', 'test-repo');
if (existsSync(testRepoPath)) {
await rm(testRepoPath, { recursive: true, force: true });
}
await mkdir(testRepoPath);
chdir(testRepoPath);
await Promise.all([clearWorkflowRuns(), createNpmPackage(), copyActionFiles(actionFileGlobPaths)]);
await git('init', '--initial-branch', 'main');
await addRemote();
await git('config', 'user.name', 'Automated Version Bump Test');
await git('config', 'user.email', 'gh-action-bump-version-test@users.noreply.github.com');
await git('add', '.');
await git('commit', '--message', 'initial commit (version 1.0.0)');
await deleteTags();
};
function createNpmPackage() {
return exec('npm', 'init', '-y');
}
async function addRemote() {
const testRepoUrl = process.env.TEST_REPO;
const username = process.env.TEST_USER;
const token = process.env.TEST_TOKEN;
const authUrl = testRepoUrl.replace(/^https:\/\//, `https://${username}:${token}@`);
await git('remote', 'add', 'origin', authUrl);
}
async function copyActionFiles(globPaths) {
const actionFolder = join(cwd(), 'action');
await mkdir(actionFolder);
const projectRoot = join(__dirname, '..', '..');
const globResults = await Promise.all(globPaths.map((path) => glob(path, { cwd: projectRoot })));
const relativeFilePaths = await Promise.all([...new Set(globResults.flat())]);
const folders = [...new Set(relativeFilePaths.map(dirname))].filter((path) => path !== '.');
if (folders.length > 0) {
await Promise.all(folders.map((folder) => mkdir(join(actionFolder, folder), { recursive: true })));
}
await Promise.all(
relativeFilePaths.map(async (path) => {
const sourcePath = join(projectRoot, path);
const fileStat = await stat(sourcePath);
if (fileStat.isFile()) {
return copyFile(sourcePath, join(actionFolder, path));
}
}),
);
}
async function deleteTags() {
const listTagsResult = await git({ suppressOutput: true }, 'ls-remote', '--tags', 'origin');
if (listTagsResult.stdout) {
const lines = listTagsResult.stdout.split('\n');
const tags = lines.map((line) => line.split('\t')[1]);
for (const tag of tags) {
await git('push', 'origin', '--delete', tag);
}
}
}