Need a more "DRY" solution for my CircleCI tests - node.js

Question

Currently I have a working circleci yaml file but it runs a separate test script for each app. It's not "DRY" and I think a better solution would be to run the test script from the root and loop over each app...but I can't figure out how to do that. I've seen some examples but anytime I try to apply them, circle can't seem to find the clover files so I'm not sure what I'm doing wrong. I'd LOVE to run everything in parallel too but my first need is to just eliminate all the extra steps.
This is the current setup:
file system structure:
- apps
- **app1**
- coverage
- clover.xml
- coverage-final.json
- src
- test-files
package.json
- **app2**
- coverage
- clover.xml
- coverage-final.json
- src
- test-files
package.json
- **app3**
- coverage
- clover.xml
- coverage-final.json
- src
- test-files
package.json
- package.json
root/package.json scripts
{
"scripts": {
"lint": "lerna run lint --concurrency=2",
"build": "lerna run build --concurrency=1 --stream",
"bootstrap:ci": "lerna bootstrap --ci --concurrency=2"
"test": "CI=true lerna run test:coverage --concurrency=2 --stream -- --coverage",
"test:app1": "cd apps/enhanced-date-editor && npm run test:coverage",
"test:app2": "cd apps/input-fields-group && npm run test:coverage",
"test:app3": "cd apps/markdown-items-list && npm run test:coverage",
},
"devDependencies": {
"lerna": "^3.22.1"
}
}
app[1/2/3]/package.json scripts
{
"scripts": {
"build": "react-scripts build",
"test": "react-scripts test",
"test:coverage": "npm t -- --coverage",
"deploy": "s3 bucket script",
},
"jest": {
"collectCoverageFrom": [
"**/*.{js,jsx,tsx}",
"!**/node_modules/**",
"!**/vendor/**"
]
}
}
circleci/config.yml
version: 2.1
executors:
node:
docker:
- image: cimg/node:14.15.4
jobs:
prepare:
executor: node
steps:
- checkout
- run:
name: Install dependencies
command: |
npm i
npm run bootstrap:ci
- persist_to_workspace:
root: .
paths:
- .
lint:
executor: node
steps:
- attach_workspace:
at: .
- run:
name: lint
command: npm run lint
test-reporter:
executor: node
steps:
- run:
name: Setup Code Climate test-reporter
command: |
# download test reporter as a static binary
mkdir -p testing-workspace/test-results/coverage
curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > testing-workspace/cc-test-reporter
chmod +x testing-workspace/cc-test-reporter
- persist_to_workspace:
root: testing-workspace
paths:
- cc-test-reporter
test-app1:
executor: node
steps:
- attach_workspace:
at: .
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: Testing and formatting for codeclimate
command: |
testing-workspace/cc-test-reporter before-build
npm run test:app1
testing-workspace/cc-test-reporter format-coverage \
"apps/app1/coverage/clover.xml" \
--input-type clover \
--output testing-workspace/test-results/coverage/coverage.app1.json
- persist_to_workspace:
root: testing-workspace
paths:
- test-results/coverage/coverage.app1.json
test-app2:
executor: node
steps:
- attach_workspace:
at: .
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: Testing and formatting for codeclimate
command: |
testing-workspace/cc-test-reporter before-build
npm run test:app2
testing-workspace/cc-test-reporter format-coverage \
"apps/app2/coverage/clover.xml" \
--input-type clover \
--output testing-workspace/test-results/coverage/coverage.app2.json
- persist_to_workspace:
root: testing-workspace
paths:
- test-results/coverage/coverage.app2.json
test-app3:
executor: node
steps:
- attach_workspace:
at: .
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: Testing and formatting for codeclimate
command: |
testing-workspace/cc-test-reporter before-build
npm run test:app3
testing-workspace/cc-test-reporter format-coverage \
"apps/app3/coverage/clover.xml" \
--input-type clover \
--output testing-workspace/test-results/coverage/coverage.app3.json
- persist_to_workspace:
root: testing-workspace
paths:
- test-results/coverage/coverage.app3.json
ship-all-the-test-results:
executor: node
steps:
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: sum coverage
command: testing-workspace/cc-test-reporter sum-coverage -o testing-workspace/test-results/coverage/coverage.total.json testing-workspace/test-results/coverage/coverage.*.json;
- run:
name: ship results to code climate
command: testing-workspace/cc-test-reporter upload-coverage --input testing-workspace/test-results/coverage/coverage.total.json
- store_artifacts:
path: testing-workspace/test-results
destination: test-results
- store_test_results:
path: testing-workspace/test-results
workflows:
build-test-deploy:
jobs:
- prepare
- lint:
requires:
- prepare
- test-reporter:
requires:
- lint
- test-app1:
requires:
- test-reporter
- test-app2:
requires:
- test-reporter
- test-app3:
requires:
- test-reporter
- ship-all-the-test-results:
requires:
- test-app1
- test-app2
- test-app3
Example of what I've tried
testing-workspace/cc-test-reporter before-build
npm run test
testing-workspace/cc-test-reporter format-coverage apps/**/coverage/clover.xml --input-type clover --output apps/**/coverage/coverage-final.json
testing-workspace/cc-test-reporter sum-coverage apps/**/coverage/coverage-final.json --output testing-workspace/test-results/coverage/codeclimate.total.json
testing-workspace/cc-test-reporter upload-coverage --input testing-workspace/test-results/coverage/coverage.total.json
testing-workspace/cc-test-reporter after-build --exit-code $?
error was git head does not match
Another Example of what I have tried
#!/bin/bash -eo pipefail
# notify codeclimate that a build is coming
testing-workspace/cc-test-reporter before-build
# run tests
# will receive a list of test files and the CircleCI CLI will split them by timing data.
TESTFILES=$(circleci tests glob "apps/**.js" | circleci tests split)
npm run test ${TESTFILES}
# format test results
testing-workspace/cc-test-reporter format-coverage "**/**/clover.xml" --input-type clover --output "apps/**/coverage/coverage-final.json"
# combine test results
testing-workspace/cc-test-reporter sum-coverage -o testing-workspace/test-results/coverage/coverage.total.json "apps/**/coverage/coverage-final.json"
# upload test results to codeclimate
testing-workspace/cc-test-reporter upload-coverage --input testing-workspace/test-results/coverage/coverage.total.json
# notify codeclimate that build is completed
testing-workspace/cc-test-reporter after-build --exit-code $?
error was ```could not find coverage file //clover.xml
could not find any files in search paths for clover. search paths were: //clover.xml, build/logs/clover.xml, clover.xml ``

Answer 1

There is no loop in CircleCI config syntax, but you can make the config more DRY by adding a parameter for the app to your test-app job.
version: 2.1
executors:
node:
docker:
- image: cimg/node:14.15.4
jobs:
prepare:
executor: node
steps:
- checkout
- run:
name: Install dependencies
command: |
npm i
npm run bootstrap:ci
- persist_to_workspace:
root: .
paths:
- .
lint:
executor: node
steps:
- attach_workspace:
at: .
- run:
name: lint
command: npm run lint
test-reporter:
executor: node
steps:
- run:
name: Setup Code Climate test-reporter
command: |
# download test reporter as a static binary
mkdir -p testing-workspace/test-results/coverage
curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > testing-workspace/cc-test-reporter
chmod +x testing-workspace/cc-test-reporter
- persist_to_workspace:
root: testing-workspace
paths:
- cc-test-reporter
test-app:
executor: node
parameters:
app_name:
type: string
steps:
- attach_workspace:
at: .
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: Testing and formatting for codeclimate
command: |
testing-workspace/cc-test-reporter before-build
npm run test:<< parameters.app_name >>
testing-workspace/cc-test-reporter format-coverage \
"apps/<< parameters.app_name >>/coverage/clover.xml" \
--input-type clover \
--output testing-workspace/test-results/coverage/coverage.<< parameters.app_name >>.json
- persist_to_workspace:
root: testing-workspace
paths:
- test-results/coverage/coverage.<< parameters.app_name >>.json
ship-all-the-test-results:
executor: node
steps:
- attach_workspace:
at: ~/project/testing-workspace
- run:
name: sum coverage
command: testing-workspace/cc-test-reporter sum-coverage -o testing-workspace/test-results/coverage/coverage.total.json testing-workspace/test-results/coverage/coverage.*.json;
- run:
name: ship results to code climate
command: testing-workspace/cc-test-reporter upload-coverage --input testing-workspace/test-results/coverage/coverage.total.json
- store_artifacts:
path: testing-workspace/test-results
destination: test-results
- store_test_results:
path: testing-workspace/test-results
workflows:
build-test-deploy:
jobs:
- prepare
- lint:
requires:
- prepare
- test-reporter:
requires:
- lint
- test-app:
name: test-app1
app_name: app1
requires:
- test-reporter
- test-app:
name: test-app2
app_name: app2
requires:
- test-reporter
- test-app:
name: test-app3
app_name: app3
requires:
- test-reporter
- ship-all-the-test-results:
requires:
- test-app1
- test-app2
- test-app3