set up end-to-end testing

.github/workflows/pull-request.yml

@@ -0,0 +1,20 @@
+name: Confirm Pull Request
+
+on:
+  pull_request:
+
+concurrency: end-to-end-test
+
+jobs:
+  end-to-end-tests:
+    name: Conduct end-to-end tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - run: npm ci
+        name: Install dependencies
+      - run: npm run end-to-end-test
+        env:
+          TEST_REPO: ${{ secrets.TEST_REPO }}
+          TEST_USER: ${{ secrets.TEST_USER }}
+          TEST_TOKEN: ${{ secrets.TEST_TOKEN }}

.gitignore

@@ -1,2 +1,4 @@
 node_modules
 .nyc_output
+/test-repo
+/.env

package.json

@@ -16,15 +16,22 @@
   "main": "index.js",
   "scripts": {
     "start": "node ./index.js",
-    "test": "jest"
+    "test": "npm run end-to-end-test",
+    "end-to-end-test": "jest --roots=tests/end-to-end --testTimeout=300000 --runInBand"
   },
   "dependencies": {
     "actions-toolkit": "^2.2.0",
     "yargs": "^15.3.1"
   },
   "devDependencies": {
-    "jest": "^25.2.7",
+    "@types/jest": "^27.0.1",
+    "dotenv": "^10.0.0",
+    "execa": "^5.1.1",
+    "jest": "^25.5.4",
+    "js-yaml": "^4.1.0",
+    "node-fetch": "^2.6.1",
+    "prettier": "^2.3.0",
     "standard": "^14.3.3",
-    "prettier": "^2.3.0"
+    "tiny-glob": "^0.2.9"
   }
 }

tests/end-to-end/actionsApi.js

@@ -0,0 +1,85 @@
+const { default: fetch } = require('node-fetch');
+
+async function clearWorkflowRuns() {
+  const runs = await getWorkflowRuns();
+  const basePath = getActionsBasePath();
+  await Promise.all(runs.map((run) => api(`${basePath}/runs/${run.id}`, { method: 'DELETE' })));
+}
+exports.clearWorkflowRuns = clearWorkflowRuns;
+
+async function getMostRecentWorkflowRun() {
+  const runs = await getWorkflowRuns();
+  if (runs.length === 0) {
+    return null;
+  }
+  const mostRecentRun = runs.reduce((previous, current) => {
+    const prevDate = new Date(previous.created_at);
+    const currDate = new Date(current.created_at);
+    if (prevDate < currDate) {
+      return current;
+    } else {
+      return previous;
+    }
+  });
+  return mostRecentRun;
+}
+exports.getMostRecentWorkflowRun = getMostRecentWorkflowRun;
+
+async function getWorkflowRun(id) {
+  const basePath = getActionsBasePath();
+  const run = await api(`${basePath}/runs/${id}`);
+  return run;
+}
+exports.getWorkflowRun = getWorkflowRun;
+
+async function getWorkflowRuns() {
+  const basePath = getActionsBasePath();
+  const result = await api(`${basePath}/runs`);
+  return result.workflow_runs || [];
+}
+
+function getActionsBasePath() {
+  const repoUrl = process.env.TEST_REPO;
+  const match = /\/([^/]*)\/([^/]*)\.git$/.exec(repoUrl);
+  const owner = match[1];
+  const repo = match[2];
+  return `repos/${owner}/${repo}/actions`;
+}
+
+const retryAttempts = 10;
+const retryInterval = 10;
+
+async function api(path, options) {
+  options = options || {};
+  const username = process.env.TEST_USER;
+  const token = process.env.TEST_TOKEN;
+  for (let attempts = 0; attempts < retryAttempts; attempts++) {
+    const response = await fetch(`https://api.github.com/${path}`, {
+      method: options.method || 'GET',
+      headers: {
+        Authorization: `Basic ${Buffer.from(`${username}:${token}`, 'ascii').toString('base64')}`,
+        accept: 'application/vnd.github.v3+json',
+        'User-Agent': username,
+        ...options.headers,
+      },
+    });
+    if (response.ok) {
+      const responseText = await response.text();
+      if (responseText.length > 0) {
+        return JSON.parse(responseText);
+      } else {
+        return;
+      }
+    } else if (response.status === 404) {
+      console.log(
+        `Received a 404 error while executing request at ${path}. Waiting ${retryInterval} seconds and retrying...`,
+      );
+      await new Promise((resolve) => setTimeout(resolve, retryInterval * 1000));
+    } else {
+      throw new Error(`${response.status} ${response.statusText}`);
+    }
+  }
+  throw new Error(
+    `Received ${retryAttempts} 404 errors in a row while executing request at ${path} in ${retryInterval}-second intervals.`,
+  );
+}

tests/end-to-end/config.yaml

@@ -0,0 +1,140 @@
+suites:
+  - name: default
+    yaml:
+      name: Bump Version
+      on:
+        push:
+      jobs:
+        bump-version:
+          runs-on: ubuntu-latest
+          steps:
+            - uses: actions/checkout@v2
+            - id: version-bump
+              uses: ./action
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    tests:
+      - message: no keywords
+        expected:
+          version: 1.0.1
+      - message: feat
+        expected:
+          version: 1.1.0
+      - message: minor
+        expected:
+          version: 1.2.0
+      - message: BREAKING CHANGE
+        expected:
+          version: 2.0.0
+      - message: major
+        expected:
+          version: 3.0.0
+      - message: pre-alpha
+        expected:
+          version: 3.0.1-alpha.0
+      - message: pre-beta
+        expected:
+          version: 3.0.1-beta.0
+      - message: pre-rc
+        expected:
+          version: 3.0.1-rc.0
+  - name: no-push
+    yaml:
+      name: Do Nothing
+      on:
+        push:
+      jobs:
+        bump-version:
+          runs-on: ubuntu-latest
+          steps:
+            - uses: actions/checkout@v2
+            - id: version-bump
+              uses: ./action
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                push: false
+    tests:
+      - message: no keywords
+        expected:
+          version: 3.0.1-rc.0
+  - name: custom-wording
+    yaml:
+      name: Bump Version (Custom Wording)
+      on:
+        push:
+      jobs:
+        bump-version:
+          runs-on: ubuntu-latest
+          steps:
+            - uses: actions/checkout@v2
+            - id: version-bump
+              uses: ./action
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                minor-wording: custom-minor
+                major-wording: custom-major
+                rc-wording: custom-pre
+    tests:
+      - message: custom-minor
+        expected:
+          version: 3.1.0
+      - message: custom-major
+        expected:
+          version: 4.0.0
+      - message: custom-pre
+        expected:
+          version: 4.0.1-pre.0
+  - name: null-default
+    yaml:
+      name: Bump Version (Default="Minor")
+      on:
+        push:
+      jobs:
+        bump-version:
+          runs-on: ubuntu-latest
+          steps:
+            - uses: actions/checkout@v2
+            - id: version-bump
+              uses: ./action
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                default: minor
+                patch-wording: patch
+    tests:
+      - message: no hint
+        expected:
+          version: 4.1.0
+      - message: patch
+        expected:
+          version: 4.1.1
+  - name: custom-tag-prefix
+    yaml:
+      name: Bump Version (Custom Tag Prefix)
+      on:
+        push:
+      jobs:
+        bump-version:
+          runs-on: ubuntu-latest
+          steps:
+            - uses: actions/checkout@v2
+            - id: version-bump
+              uses: ./action
+              env:
+                GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              with:
+                tag-prefix: 'v'
+    tests:
+      - message: no keywords
+        expected:
+          version: 4.1.2
+          tag: v4.1.2
+actionFiles:
+  - index.js
+  - Dockerfile
+  - package.json
+  - package-lock.json
+  - action.yml
+image: catthehacker/ubuntu:act-latest

tests/end-to-end/exec.js

@@ -0,0 +1,17 @@
+const execa = require('execa');
+
+module.exports = async function exec(command, options, ...params) {
+  let suppressOutput;
+  if (typeof options === 'object') {
+    suppressOutput = options.suppressOutput;
+  } else {
+    params.unshift(options);
+    suppressOutput = false;
+  }
+  const subprocess = execa(command, params);
+  if (!suppressOutput) {
+    subprocess.stdout.pipe(process.stdout);
+  }
+  subprocess.stderr.pipe(process.stderr);
+  return await subprocess;
+};

tests/end-to-end/git.js

@@ -0,0 +1,20 @@
+const exec = require('./exec');
+
+function git(options, ...params) {
+  return exec('git', options, ...params);
+}
+
+let firstCommit = true;
+function push() {
+  if (firstCommit) {
+    firstCommit = false;
+    return git('push', '--force', '--set-upstream', 'origin', 'main');
+  } else {
+    return git('push');
+  }
+}
+
+module.exports = {
+  push,
+  default: git,
+};

tests/end-to-end/index.test.js

@@ -0,0 +1,122 @@
+const dotenv = require('dotenv');
+const setupTestRepo = require('./setupTestRepo');
+const yaml = require('js-yaml');
+const { readFileSync } = require('fs');
+const { writeFile, readFile, mkdir } = require('fs/promises');
+const { resolve, join } = require('path');
+const { cwd } = require('process');
+const { default: git, push: gitPush } = require('./git');
+const { getMostRecentWorkflowRun, getWorkflowRun } = require('./actionsApi');
+
+dotenv.config();
+
+const config = getTestConfig();
+
+beforeAll(() => setupTestRepo(config.actionFiles));
+
+config.suites.forEach((suite) => {
+  const suiteYaml = yaml.dump(suite.yaml);
+  describe(suite.name, () => {
+    beforeAll(async () => {
+      const pushYamlPath = join('.github', 'workflows', 'push.yml');
+      await mkdir(join(cwd(), '.github', 'workflows'), { recursive: true });
+      await writeFile(join(cwd(), pushYamlPath), suiteYaml);
+      await git('add', pushYamlPath);
+    });
+    suite.tests.forEach((commit) => {
+      test(commit.message, async () => {
+        await generateReadMe(commit, suiteYaml);
+        await git('commit', '--message', commit.message);
+
+        const mostRecentDate = await getMostRecentWorkflowRunDate();
+        await gitPush();
+
+        const completedRun = await getCompletedRunAfter(mostRecentDate);
+        expect(completedRun.conclusion).toBe('success');
+
+        await git('pull');
+        await assertExpectation(commit.expected);
+      });
+    });
+  });
+});
+
+function getTestConfig() {
+  const path = resolve(__dirname, './config.yaml');
+  const buffer = readFileSync(path);
+  const contents = buffer.toString();
+  const config = yaml.load(contents);
+  return config;
+}
+
+async function generateReadMe(commit, suiteYaml) {
+  const readmePath = 'README.md';
+  const readMeContents = [
+    '# Test Details',
+    '## .github/workflows/push.yml',
+    '```YAML',
+    yaml.dump(suiteYaml),
+    '```',
+    '## Message',
+    commit.message,
+    '## Expectation',
+    `**Version:** ${commit.expected.version}`,
+  ].join('\n');
+  await writeFile(join(cwd(), readmePath), readMeContents);
+  await git('add', readmePath);
+}
+
+async function getCompletedRunAfter(date) {
+  const run = await pollFor(getMostRecentWorkflowRun, (run) => run !== null && new Date(run.created_at) > date);
+  const completedRun = await pollFor(
+    () => getWorkflowRun(run.id),
+    (run) => run.status === 'completed',
+  );
+  return completedRun;
+}
+
+function pollFor(getResult, validateResult) {
+  return new Promise((resolve, reject) => {
+    pollAndRetry();
+
+    async function pollAndRetry() {
+      try {
+        const result = await getResult();
+        if (validateResult(result)) {
+          resolve(result);
+        } else {
+          setTimeout(pollAndRetry, 1000);
+        }
+      } catch (error) {
+        reject(error);
+      }
+    }
+  });
+}
+
+async function getMostRecentWorkflowRunDate() {
+  const run = await getMostRecentWorkflowRun();
+  const date = run === null ? new Date(0) : new Date(run.created_at);
+  return date;
+}
+
+async function assertExpectation({ version: expectedVersion, tag: expectedTag }) {
+  if (expectedTag === undefined) {
+    expectedTag = expectedVersion;
+  }
+  const [packageVersion, latestTag] = await Promise.all([getPackageJsonVersion(), getLatestTag()]);
+  expect(packageVersion).toBe(expectedVersion);
+  expect(latestTag).toBe(expectedTag);
+}
+
+async function getPackageJsonVersion() {
+  const path = join(cwd(), 'package.json');
+  const contents = await readFile(path);
+  const json = JSON.parse(contents);
+  return json.version;
+}
+
+async function getLatestTag() {
+  const result = await git({ suppressOutput: true }, 'describe', '--tags', '--abbrev=0');
+  return result.stdout;
+}

tests/end-to-end/setupTestRepo.js

@@ -0,0 +1,69 @@
+const { existsSync } = require('fs');
+const { rm, mkdir, copyFile, stat } = require('fs/promises');
+const { chdir, cwd } = require('process');
+const { resolve, join, dirname } = require('path');
+const exec = require('./exec');
+const { default: git } = require('./git');
+const glob = require('tiny-glob');
+const { clearWorkflowRuns } = require('./actionsApi');
+
+module.exports = async function setupTestRepo(actionFileGlobPaths) {
+  const testRepoPath = resolve(__dirname, '..', '..', 'test-repo');
+  if (existsSync(testRepoPath)) {
+    await rm(testRepoPath, { recursive: true, force: true });
+  }
+  await mkdir(testRepoPath);
+  chdir(testRepoPath);
+  await Promise.all([clearWorkflowRuns(), createNpmPackage(), copyActionFiles(actionFileGlobPaths)]);
+  await git('init', '--initial-branch', 'main');
+  await addRemote();
+  await git('config', 'user.name', 'Automated Version Bump Test');
+  await git('config', 'user.email', 'gh-action-bump-version-test@users.noreply.github.com');
+  await git('add', '.');
+  await git('commit', '--message', 'initial commit (version 1.0.0)');
+  await deleteTags();
+};
+
+function createNpmPackage() {
+  return exec('npm', 'init', '-y');
+}
+
+async function addRemote() {
+  const testRepoUrl = process.env.TEST_REPO;
+  const username = process.env.TEST_USER;
+  const token = process.env.TEST_TOKEN;
+  const authUrl = testRepoUrl.replace(/^https:\/\//, `https://${username}:${token}@`);
+  await git('remote', 'add', 'origin', authUrl);
+}
+
+async function copyActionFiles(globPaths) {
+  const actionFolder = join(cwd(), 'action');
+  await mkdir(actionFolder);
+  const projectRoot = join(__dirname, '..', '..');
+  const globResults = await Promise.all(globPaths.map((path) => glob(path, { cwd: projectRoot })));
+  const relativeFilePaths = await Promise.all([...new Set(globResults.flat())]);
+  const folders = [...new Set(relativeFilePaths.map(dirname))].filter((path) => path !== '.');
+  if (folders.length > 0) {
+    await Promise.all(folders.map((folder) => mkdir(join(actionFolder, folder), { recursive: true })));
+  }
+  await Promise.all(
+    relativeFilePaths.map(async (path) => {
+      const sourcePath = join(projectRoot, path);
+      const fileStat = await stat(sourcePath);
+      if (fileStat.isFile()) {
+        return copyFile(sourcePath, join(actionFolder, path));
+      }
+    }),
+  );
+}
+
+async function deleteTags() {
+  const listTagsResult = await git({ suppressOutput: true }, 'ls-remote', '--tags', 'origin');
+  if (listTagsResult.stdout) {
+    const lines = listTagsResult.stdout.split('\n');
+    const tags = lines.map((line) => line.split('\t')[1]);
+    for (const tag of tags) {
+      await git('push', 'origin', '--delete', tag);
+    }
+  }
+}