The usual test flow, for testing most addons, is using acceptance tests with the dummy app, and testing functionality of your components etc. via integration and unit tests. I was not aware there was even an option to run a real app, consume the addon and ensure the addon interacts with the app as intended, until I was introduced to ember-cli-addon-tests.
I have been working on adding tests to ember-cli-code-coverage using ember-cli-addon-tests, and I wanted to document how we are testing running our addon against a normal app, an in-repo addon, and an in-repo engine in case someone else has a need to test how their addon behaves in these situations.
Testing with an App
Testing with a vanilla app is the simplest case, and if you do not want to
generate any files to test against, there is no need to create any fixtures. If
you do want to create some files to test against, like we did in
ember-cli-code-coverage, you can easily create them by putting them in
test/fixtures/<your app name>.
For us, we started with:
mkdir test/fixtures/my-appWe then had a couple utils we wanted to include, to see if they were covered, so
we copied them into their places in fixtures,
test/fixtures/my-app/app/utils/my-covered-util.js and
test/fixtures/my-app/app/utils/my-uncovered-util.js.
Now that we have our fixtures setup, it is time to actually use
ember-cli-addon-tests to spin up the app and test our addon. To do this, we need
to setup our first mocha test. It will live in
test/integration/app-coverage-test.js.
We first need to require in the test app and create a new instance of the app
in our beforeEach. This will setup a new app for us to test against for each
of our tests. You can also specify emberVersion to choose which version of
Ember the app should run.
// test/integration/app-coverage-test.js
// require in the test app
const AddonTestApp = require('ember-cli-addon-tests').AddonTestApp;
let app;
describe('app coverage generation', function() {
this.timeout(10000000);
beforeEach(function() {
app = new AddonTestApp();
return app.create('my-app', {
emberVersion: '2.16.0'
})
...Now that we have instantiated our test app, we can do things like edit the
package.json, run npm install, or whatever we need to do to get the app in the
right state to test our addon. In our case, we needed to add
ember-exam so we could support
parallel testing, and we also needed to remove our own addon, and add it back to
get around symlink bugs.
...
.then(() => {
// Add ember-exam as a dep and remove the addon we are testing, so we can fix the symlink issue.
app.editPackageJSON(pkg => {
pkg.devDependencies['ember-exam'] = '0.7.0';
// Temporarily remove the addon before install to work around https://github.com/tomdale/ember-cli-addon-tests/issues/176
delete pkg.devDependencies['ember-cli-code-coverage'];});
return app.run('npm', 'install').then(() => {
app.editPackageJSON(pkg => {
pkg.devDependencies['ember-cli-code-coverage'] = '*';
});
let addonPath = path.join(app.path, 'node_modules', 'ember-cli-code-coverage');
fs.removeSync(addonPath);
// Make sure we are symlinked to our actual addon for testing and not pointing to an installed version from npm
fs.ensureSymlinkSync(process.cwd(), addonPath);
// Remove old coverage directories from previous runs
return rimraf(`${app.path}/coverage*`);
});
});
afterEach(function() {
// Remove coverage config to clean up
return RSVP.all([
rimraf(`${app.path}/config/coverage.js`)
]);
});After finishing all this setup, we are ready to run our test app and check its
output to see if our addon is working! You can run normal commands, just like
you would on the command line normally, like app.run('ember', 'test').
it('runs coverage when env var is set', function () {
// Make sure coverage did not already exist. `app.path` is the path to our test app's directory
expect(dir(`${app.path}/coverage`)).to.not.exist;
process.env.COVERAGE = true;
// Run `ember test` with COVERAGE=true to generate code coverage in our test app
return app.run('ember', 'test').then(function () {
// Check the files in our test app, make sure they exist, and make sure coverage totals are correct.
expect(file(`${app.path}/coverage/lcov-report/index.html`)).to.not.be.empty;
expect(file(`${app.path}/coverage/index.html`)).to.not.be.empty;
var summary = fs.readJSONSync(`${app.path}/coverage/coverage-summary.json`);
expect(summary.total.lines.pct).to.equal(83.33);
});
});Putting all of this together you get a test file like this:
const AddonTestApp = require('ember-cli-addon-tests').AddonTestApp;
chai.use(chaiFiles);
let app;
describe('app coverage generation', function () {
this.timeout(10000000);
beforeEach(function () {
app = new AddonTestApp();
return app
.create('my-app', {
emberVersion: '2.16.0'
})
.then(() => {
app.editPackageJSON((pkg) => {
pkg.devDependencies['ember-exam'] = '0.7.0';
// Temporarily remove the addon before install to work around https://github.com/tomdale/ember-cli-addon-tests/issues/176
delete pkg.devDependencies['ember-cli-code-coverage'];
});
return app.run('npm', 'install').then(() => {
app.editPackageJSON((pkg) => {
pkg.devDependencies['ember-cli-code-coverage'] = '*';
});
let addonPath = path.join(
app.path,
'node_modules',
'ember-cli-code-coverage'
);
fs.removeSync(addonPath);
fs.ensureSymlinkSync(process.cwd(), addonPath);
return rimraf(`${app.path}/coverage*`);
});
});
});
afterEach(function () {
return RSVP.all([rimraf(`${app.path}/config/coverage.js`)]);
});
it('runs coverage when env var is set', function () {
expect(dir(`${app.path}/coverage`)).to.not.exist;
process.env.COVERAGE = true;
return app.run('ember', 'test').then(function () {
expect(
file(`${app.path}/coverage/lcov-report/index.html`)
).to.not.be.empty;
expect(file(`${app.path}/coverage/index.html`)).to.not.be.empty;
var summary = fs.readJSONSync(
`${app.path}/coverage/coverage-summary.json`
);
expect(summary.total.lines.pct).to.equal(83.33);
});
});
});This is not an out of the box working example that you can copy and paste. Things have been left out, for the sake of brevity. To see the actual tests and fixtures, please go here.
Testing with an in-repo Addon
Testing with an in-repo addon is similar to testing with a vanilla app, but we will need a helper to generate things for the in-repo addon. Luckily, @rwjblue has us covered and created these awesome helpers in ember-engines, in-repo-addon.js and in-repo-engine.js which we shamelessly borrowed and used to test against our own in-repo addons and in repo engines.
The basic setup is the same as a vanilla app. You will create any fixtures you
want in your app, and any fixtures you want in your in-repo addon under
lib/<your in-repo addon name> and do the same basic setup of your app in
beforeEach, but the main differences come when you do your actual test. Before
running your commands with the app you generated, you will want to setup the
in-repo addon.
const AddonTestApp = require('ember-cli-addon-tests').AddonTestApp;
const InRepoAddon = require('../helpers/in-repo-addon');
chai.use(chaiFiles);
let app;
describe('in-repo addon coverage generation', function () {
this.timeout(10000000);
beforeEach(function () {
// Basically the same as a vanilla app
});
afterEach(function () {
// Basically the same as a vanilla app
});
it(
'runs coverage on in-repo addon',
co.wrap(function* () {
// Generate your in-repo addon inside your test app
let addon = yield InRepoAddon.generate(app, 'my-in-repo-addon');
// Make sure your in-repo addon has `ember-cli-babel` as a dep
addon.editPackageJSON(
(pkg) => (pkg.dependencies = { 'ember-cli-babel': '*' })
);
// Make sure coverage did not already exist. `app.path` is the path to our test app's directory
expect(dir(`${app.path}/coverage`)).to.not.exist;
process.env.COVERAGE = true;
// Run `ember test` with COVERAGE=true just like before, but now it should include coverage for our in-repo addon
return app.run('ember', 'test').then(function () {
expect(
file(`${app.path}/coverage/lcov-report/index.html`)
).to.not.be.empty;
expect(file(`${app.path}/coverage/index.html`)).to.not.be.empty;
const summary = fs.readJSONSync(
`${app.path}/coverage/coverage-summary.json`
);
expect(summary.total.lines.pct).to.equal(50);
expect(
summary['app/utils/my-covered-util-app.js'].lines.total
).to.equal(1);
// Check that lib/my-in-repo-addon/utils/my-covered-utill is 1 line and that 1 line is covered
expect(
summary['lib/my-in-repo-addon/addon/utils/my-covered-util.js'].lines
.total
).to.equal(1);
expect(
summary['lib/my-in-repo-addon/addon/utils/my-covered-util.js'].lines
.covered
).to.equal(1);
// Check that lib/my-in-repo-addon/utils/my-uncovered-utill is 1 line and that 0 lines are covered
expect(
summary['lib/my-in-repo-addon/addon/utils/my-uncovered-util.js'].lines
.total
).to.equal(1);
expect(
summary['lib/my-in-repo-addon/addon/utils/my-uncovered-util.js'].lines
.covered
).to.equal(0);
// Check that lib/my-in-repo-addon/addon-test-support/uncovered-test-support is 4 lines and that 0 lines are covered
expect(
summary[
'lib/my-in-repo-addon/addon-test-support/uncovered-test-support.js'
].lines.total
).to.equal(4);
expect(
summary[
'lib/my-in-repo-addon/addon-test-support/uncovered-test-support.js'
].lines.covered
).to.equal(0);
});
})
);
});Since we now see there is coverage info, we know our in-repo addon was generated correctly and is reporting in the coverage report. Again, these are not examples you can copy/paste, but meant to illustrate the setup steps. The full tests can be found here.
Testing with an in-repo Engine
Testing with an in-repo engine is almost identical to testing with an in-repo addon, but it uses a different helper, and your engine will have a slightly different file structure.
You again start by generating your fixtures. If you do not want anything specific, just a vanilla app and vanilla in-repo engine, you do not need to create any fixtures, but if you want to test specific files, just create them in their normal app or in-repo engine folder structure and the tests will use the ones you provide.
After you have your fixtures setup, you will essentially do the same thing we
did above in the in-repo addon tests. The only difference is you will use the
in-repo-engine helper in your test.
const InRepoEngine = require('../helpers/in-repo-engine');You will also need to add two deps to your engine’s package.json, and you may or
may not want to enable lazy loading.
it(
'runs coverage on in-repo engine',
co.wrap(function* () {
// We use the InRepoEngine helper here
let engine = yield InRepoEngine.generate(app, 'my-in-repo-engine', {
// We have lazy loading disabled
lazy: false
});
// Our engine needs both babel and htmlbars, since there are templates generated in a default engine
engine.editPackageJSON(
(pkg) =>
(pkg.dependencies = {
'ember-cli-babel': '*',
'ember-cli-htmlbars': '*'
})
);
expect(dir(`${app.path}/coverage`)).to.not.exist;
process.env.COVERAGE = true;
return app.run('ember', 'test').then(function () {
expect(
file(`${app.path}/coverage/lcov-report/index.html`)
).to.not.be.empty;
expect(file(`${app.path}/coverage/index.html`)).to.not.be.empty;
const summary = fs.readJSONSync(
`${app.path}/coverage/coverage-summary.json`
);
expect(summary.total.lines.pct).to.equal(75);
expect(summary['app/utils/my-covered-util-app.js'].lines.total).to.equal(
1
);
// Check that lib/my-in-repo-engine/utils/my-covered-utill is 1 line and that 1 line is covered
expect(
summary['lib/my-in-repo-engine/addon/utils/my-covered-util.js'].lines
.total
).to.equal(1);
expect(
summary['lib/my-in-repo-engine/addon/utils/my-covered-util.js'].lines
.covered
).to.equal(1);
// Check that lib/my-in-repo-engine/utils/my-uncovered-utill is 1 line and that 0 lines are covered
expect(
summary['lib/my-in-repo-engine/addon/utils/my-uncovered-util.js'].lines
.total
).to.equal(1);
expect(
summary['lib/my-in-repo-engine/addon/utils/my-uncovered-util.js'].lines
.covered
).to.equal(0);
});
})
);As you can see, this is pretty much the same. Hopefully this will help some addon authors with testing their addons against real apps. I had no idea how to accomplish this programmatically before, but this makes it quite nice and easy to test with!