Add integration tests against crates in the rust-lang-nursery

This commit adds integration tests against some crates in the
nursery.

Each integration test is added as a separate build-bot, where
the rust-lang-nursery/${CRATE} is first downloaded and its tests run.
Afterwards, `cargo fmt --all` is applied to the crate, and the tests
are re-run. If the tests fail after formatting, the integration test
fails.

The crates that currently fail are added as allowed-to-fail, but the
intent is that either these crates or rustfmt should be fixed such
that the tests pass.

.travis.yml

@@ -1,21 +1,11 @@
 sudo: false
 language: rust
-rust:
-# - stable
-# - beta
- - nightly
-os:
- - linux
- - osx
+rust: nightly
+os: linux
 cache:
  directories:
   - $HOME/.cargo
 
-matrix:
-  include:
-    # Make sure tests will pass on beta
-    - env: CFG_RELEASE_CHANNEL=beta
-
 addons:
   apt:
     packages:
@@ -23,25 +13,66 @@ addons:
     - libelf-dev
     - libdw-dev
 
+matrix:
+  include:
+    - env: DEPLOY=LINUX
+    - env: CFG_RELEASE_CHANNEL=beta
+    - os: osx
+    - env: INTEGRATION=cargo
+    - env: INTEGRATION=rust-clippy
+    - env: INTEGRATION=mdbook
+    - env: INTEGRATION=stdsimd
+    - env: INTEGRATION=rust-semverver
+    - env: INTEGRATION=chalk
+    - env: INTEGRATION=crater
+    - env: INTEGRATION=futures-rs
+    - env: INTEGRATION=rand
+    - env: INTEGRATION=failure
+    - env: INTEGRATION=glob
+    - env: INTEGRATION=error-chain
+    - env: INTEGRATION=tempdir
+    - env: INTEGRATION=bitflags
+    - env: INTEGRATION=log
+  allow_failures:
+    - env: INTEGRATION=cargo
+    - env: INTEGRATION=stdsimd
+    - env: INTEGRATION=mdbook
+    - env: INTEGRATION=crater
+    - env: INTEGRATION=rust-semverver
+    - env: INTEGRATION=rust-clippy
+    - env: INTEGRATION=chalk
+    - env: INTEGRATION=bitflags
+    - env: INTEGRATION=error-chain
+    - env: INTEGRATION=failure
+    - env: INTEGRATION=futures-rs
+    - env: INTEGRATION=log
+    - env: INTEGRATION=rand
+
 before_script:
 - |
-  if [ $TRAVIS_OS_NAME = 'osx' ]; then
-    virtualenv env &&
-    source env/bin/activate &&
-    python --version &&
-    pip install 'travis-cargo<0.2'
-  else
-    pip install 'travis-cargo<0.2' --user &&
-    export PATH="$(python -m site --user-base)/bin:$PATH"
+  if [ -z ${INTEGRATION} ]; then
+    if [ $TRAVIS_OS_NAME = 'osx' ]; then
+      virtualenv env &&
+      source env/bin/activate &&
+      python --version &&
+      pip install 'travis-cargo<0.2'
+    else
+      pip install 'travis-cargo<0.2' --user &&
+      export PATH="$(python -m site --user-base)/bin:$PATH"
+    fi
   fi
 
 script:
-- |
-  cargo build &&
-  cargo test
+  - |
+    if [ -z ${INTEGRATION} ]; then
+      cargo build
+      cargo test
+    else
+      ./ci/integration.sh
+    fi
 
 after_success:
-- travis-cargo coveralls --no-sudo
+- if [ -z ${INTEGRATION} ]; then travis-cargo coveralls --no-sudo; fi
 
 before_deploy:
   # TODO: cross build
@@ -57,5 +88,5 @@ deploy:
   on:
     repo: nrc/rustfmt
     tags: true
-    condition: "$TRAVIS_OS_NAME = linux"
+    condition: "$DEPLOY = LINUX"
   skip_cleanup: true

ci/integration.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -ex
+
+: ${INTEGRATION?"The INTEGRATION environment variable must be set."}
+
+# FIXME: this is causing the build to fail when rustfmt is found in .cargo/bin
+# but cargo-fmt is not found.
+#
+# `which rustfmt` fails if rustfmt is not found. Since we don't install
+# `rustfmt` via `rustup`, this is the case unless we manually install it. Once
+# that happens, `cargo install --force` will be called, which installs
+# `rustfmt`, `cargo-fmt`, etc to `~/.cargo/bin`. This directory is cached by
+# travis (see `.travis.yml`'s "cache" key), such that build-bots that arrive
+# here after the first installation will find `rustfmt` and won't need to build
+# it again.
+#
+# which rustfmt || cargo install --force
+cargo install --force
+
+echo "Integration tests for: ${INTEGRATION}"
+
+function check_fmt {
+    cargo fmt --all -v -- --error-on-unformatted &> rustfmt_output
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+    cat rustfmt_output
+    ! cat rustfmt_output | grep -q "internal error"
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+    ! cat rustfmt_output | grep -q "warning"
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+    ! cat rustfmt_output | grep -q "Warning"
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+    cargo test --all
+    if [[ $? != 0 ]]; then
+        return $?
+    fi
+}
+
+function check {
+    cargo test --all
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+    check_fmt
+    if [[ $? != 0 ]]; then
+        return 1
+    fi
+}
+
+case ${INTEGRATION} in
+    cargo)
+        git clone https://github.com/rust-lang/${INTEGRATION}.git
+        cd ${INTEGRATION}
+        export CFG_DISABLE_CROSS_TESTS=1
+        check
+        cd -
+        ;;
+    failure)
+        git clone https://github.com/rust-lang-nursery/${INTEGRATION}.git
+        cd ${INTEGRATION}/failure-1.X
+        check
+        cd -
+        ;;
+    *)
+        git clone https://github.com/rust-lang-nursery/${INTEGRATION}.git
+        cd ${INTEGRATION}
+        check
+        cd -
+        ;;
+esac