Merge master into features

.coveragerc

@@ -1,9 +1,18 @@
 [run]
-source = pytest,_pytest,testing/
+include =
+  */src/*
+  testing/*
+  */lib/python*/site-packages/_pytest/*
+  */lib/python*/site-packages/pytest.py
+  */pypy*/site-packages/_pytest/*
+  */pypy*/site-packages/pytest.py
+  *\Lib\site-packages\_pytest\*
+  *\Lib\site-packages\pytest.py
 parallel = 1
 branch = 1
 
 [paths]
 source = src/
-  .tox/*/lib/python*/site-packages/
-  .tox\*\Lib\site-packages\
+  */lib/python*/site-packages/
+  */pypy*/site-packages/
+  *\Lib\site-packages\

.travis.yml

@@ -9,69 +9,86 @@ stages:
 python:
   - '3.7'
 install:
-  - pip install --upgrade --pre tox
-env:
-  matrix:
-    - TOXENV=py27
-    # Specialized factors for py27.
-    - TOXENV=py27-nobyte
-    - TOXENV=py27-xdist
-    - TOXENV=py27-pluggymaster
-    # Specialized factors for py37.
-    - TOXENV=py37-pexpect,py37-trial,py37-numpy
-    - TOXENV=py37-pluggymaster
-    - TOXENV=py37-freeze PYTEST_NO_COVERAGE=1
-
-matrix:
-  allow_failures:
-    - python: '3.8-dev'
-      env: TOXENV=py38-xdist
+  - python -m pip install --upgrade --pre tox
 
 jobs:
   include:
+    # OSX tests - first (in test stage), since they are the slower ones.
+    - &test-macos
+      # NOTE: (tests with) pexpect appear to be buggy on Travis,
+      #       at least with coverage.
+      #       Log: https://travis-ci.org/pytest-dev/pytest/jobs/500358864
+      os: osx
+      osx_image: xcode10.1
+      language: generic
+      # Coverage for:
+      # - py2 with symlink in test_cmdline_python_package_symlink.
+      env: TOXENV=py27-xdist PYTEST_COVERAGE=1
+      before_install:
+        - python -V
+        - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 27
+    - <<: *test-macos
+      env: TOXENV=py37-xdist
+      before_install:
+        - which python3
+        - python3 -V
+        - ln -sfn "$(which python3)" /usr/local/bin/python
+        - python -V
+        - test $(python -c 'import sys; print("%d%d" % sys.version_info[0:2])') = 37
+
+    # Full run of latest (major) supported versions, without xdist.
+    - env: TOXENV=py27
+      python: '2.7'
+    - env: TOXENV=py37
+      python: '3.7'
+
     # Coverage tracking is slow with pypy, skip it.
-    - env: TOXENV=pypy-xdist PYTEST_NO_COVERAGE=1
+    - env: TOXENV=pypy-xdist
       python: 'pypy2.7-6.0'
-    - env: TOXENV=pypy3-xdist PYTEST_NO_COVERAGE=1
+    - env: TOXENV=pypy3-xdist
       python: 'pypy3.5-6.0'
 
     - env: TOXENV=py34-xdist
       python: '3.4'
     - env: TOXENV=py35-xdist
       python: '3.5'
-    - env: TOXENV=py36-xdist
-      python: '3.6'
-    - env: TOXENV=py37
-    - &test-macos
-      language: generic
-      os: osx
-      osx_image: xcode9.4
-      sudo: required
-      install:
-        - python -m pip install --pre tox
-      env: TOXENV=py27-xdist
-    - <<: *test-macos
-      env: TOXENV=py37-xdist
-      before_install:
-        - brew update
-        - brew upgrade python
-        - brew unlink python
-        - brew link python
+
+    # Coverage for:
+    # - pytester's LsofFdLeakChecker
+    # - TestArgComplete (linux only)
+    # - numpy
+    - env: TOXENV=py37-lsof-numpy-xdist PYTEST_COVERAGE=1
+
+    # Specialized factors for py27.
+    - env: TOXENV=py27-nobyte-numpy-xdist
+      python: '2.7'
+    - env: TOXENV=py27-pluggymaster-xdist
+      python: '2.7'
+
+    # Specialized factors for py37.
+    # Coverage for:
+    # - test_sys_breakpoint_interception (via pexpect).
+    - env: TOXENV=py37-pexpect,py37-trial PYTEST_COVERAGE=1
+    - env: TOXENV=py37-pluggymaster-xdist
+    - env: TOXENV=py37-freeze
 
     # Jobs only run via Travis cron jobs (currently daily).
     - env: TOXENV=py38-xdist
       python: '3.8-dev'
       if: type = cron
 
     - stage: baseline
-      env: TOXENV=py27-pexpect,py27-trial,py27-numpy
-    - env: TOXENV=py37-xdist
-    - env: TOXENV=linting,docs,doctesting
-      python: '3.7'
+    # Coverage for:
+    # - _pytest.unittest._handle_skip (via pexpect).
+      env: TOXENV=py27-pexpect,py27-trial PYTEST_COVERAGE=1
+      python: '2.7'
+    # Use py36 here for faster baseline.
+    - env: TOXENV=py36-xdist
+      python: '3.6'
+    - env: TOXENV=linting,docs,doctesting PYTEST_COVERAGE=1
 
     - stage: deploy
       python: '3.6'
-      env: PYTEST_NO_COVERAGE=1
       install: pip install -U setuptools setuptools_scm
       script: skip
       deploy:
@@ -85,9 +102,19 @@ jobs:
           tags: true
           repo: pytest-dev/pytest
 
+matrix:
+  allow_failures:
+    - python: '3.8-dev'
+      env: TOXENV=py38-xdist
+
 before_script:
   - |
-    if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then
+    # Do not (re-)upload coverage with cron runs.
+    if [[ "$TRAVIS_EVENT_TYPE" = cron ]]; then
+      PYTEST_COVERAGE=0
+    fi
+  - |
+    if [[ "$PYTEST_COVERAGE" = 1 ]]; then
       export COVERAGE_FILE="$PWD/.coverage"
       export COVERAGE_PROCESS_START="$PWD/.coveragerc"
       export _PYTEST_TOX_COVERAGE_RUN="coverage run -m"
@@ -98,14 +125,14 @@ script: tox --recreate
 
 after_success:
   - |
-    if [[ "$PYTEST_NO_COVERAGE" != 1 ]]; then
+    if [[ "$PYTEST_COVERAGE" = 1 ]]; then
       set -e
       # Add last TOXENV to $PATH.
       PATH="$PWD/.tox/${TOXENV##*,}/bin:$PATH"
       coverage combine
-      coverage xml --ignore-errors
-      coverage report -m --ignore-errors
-      bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -F $TRAVIS_OS_NAME
+      coverage xml
+      coverage report -m
+      bash <(curl -s https://codecov.io/bash) -Z -X gcov -X coveragepy -X search -X xcode -X gcovout -X fix -f coverage.xml -n $TOXENV-$TRAVIS_OS_NAME
     fi
 
 notifications:

README.rst

@@ -22,8 +22,8 @@
 .. image:: https://travis-ci.org/pytest-dev/pytest.svg?branch=master
     :target: https://travis-ci.org/pytest-dev/pytest
 
-.. image:: https://ci.appveyor.com/api/projects/status/mrgbjaua7t33pg6b?svg=true
-    :target: https://ci.appveyor.com/project/pytestbot/pytest
+.. image:: https://dev.azure.com/pytest-dev/pytest/_apis/build/status/pytest-CI?branchName=master
+    :target: https://dev.azure.com/pytest-dev/pytest
 
 .. image:: https://img.shields.io/badge/code%20style-black-000000.svg
     :target: https://github.com/ambv/black

azure-pipelines.yml

@@ -6,6 +6,9 @@ variables:
   PYTEST_ADDOPTS: "--junitxml=build/test-results/$(tox.env).xml"
   python.needs_vc: False
   python.exe: "python"
+  COVERAGE_FILE: "$(Build.Repository.LocalPath)/.coverage"
+  COVERAGE_PROCESS_START: "$(Build.Repository.LocalPath)/.coveragerc"
+  PYTEST_COVERAGE: '0'
 
 jobs:
 
@@ -17,47 +20,68 @@ jobs:
       py27:
         python.version: '2.7'
         tox.env: 'py27'
-      py27-xdist:
+      py27-nobyte-lsof-numpy:
         python.version: '2.7'
-        tox.env: 'py27-xdist'
-      py27-numpy/nobyte:
-        python.version: '2.7'
-        tox.env: 'py27-numpy,py27-nobyte'
+        tox.env: 'py27-lsof-nobyte-numpy'
+        # Coverage for:
+        # - test_supports_breakpoint_module_global
+        # - test_terminal_reporter_writer_attr (without xdist)
+        # - "if write" branch in _pytest.assertion.rewrite
+        # - numpy
+        # - pytester's LsofFdLeakChecker (being skipped)
+        PYTEST_COVERAGE: '1'
       py27-trial:
         python.version: '2.7'
         tox.env: 'py27-trial'
         python.needs_vc: True
-      py27-pluggymaster:
+      py27-pluggymaster-xdist:
         python.version: '2.7'
-        tox.env: 'pluggymaster'
+        tox.env: 'py27-pluggymaster-xdist'
+        # Coverage for:
+        # - except-IOError in _attempt_to_close_capture_file for py2.
+        #   Also seen with py27-nobyte (using xdist), and py27-xdist.
+        #   But no exception with py27-pexpect,py27-trial,py27-numpy.
+        PYTEST_COVERAGE: '1'
       pypy:
         python.version: 'pypy'
         tox.env: 'pypy'
         python.exe: 'pypy'
-      py34:
+      # NOTE: pypy3 fails to install pip currently due to an interal error.
+      # pypy3:
+      #   python.version: 'pypy3'
+      #   tox.env: 'pypy3'
+      #   python.exe: 'pypy3'
+      py34-xdist:
         python.version: '3.4'
-        tox.env: 'py34'
-      py35:
+        tox.env: 'py34-xdist'
+        # Coverage for:
+        # - _pytest.compat._bytes_to_ascii
+        PYTEST_COVERAGE: '1'
+      py35-xdist:
         python.version: '3.5'
-        tox.env: 'py35'
-      py36:
+        tox.env: 'py35-xdist'
+        # Coverage for:
+        # - test_supports_breakpoint_module_global
+        PYTEST_COVERAGE: '1'
+      py36-xdist:
         python.version: '3.6'
-        tox.env: 'py36'
+        tox.env: 'py36-xdist'
       py37:
         python.version: '3.7'
         tox.env: 'py37'
+        # Coverage for:
+        # - _py36_windowsconsoleio_workaround (with py36+)
+        # - test_request_garbage (no xdist)
+        PYTEST_COVERAGE: '1'
       py37-linting/docs/doctesting:
         python.version: '3.7'
         tox.env: 'linting,docs,doctesting'
-      py37-xdist:
-        python.version: '3.7'
-        tox.env: 'py37-xdist'
       py37-trial/numpy:
         python.version: '3.7'
         tox.env: 'py37-trial,py37-numpy'
-      py37-pluggymaster:
+      py37-pluggymaster-xdist:
         python.version: '3.7'
-        tox.env: 'py37-pluggymaster'
+        tox.env: 'py37-pluggymaster-xdist'
     maxParallel: 10
 
   steps:
@@ -91,11 +115,21 @@ jobs:
   - script: $(python.exe) -m pip install --upgrade pip && $(python.exe) -m pip install tox
     displayName: 'Install tox'
 
-  - script: $(python.exe) -m tox -e $(tox.env)
+  - script: |
+      call scripts/setup-coverage-vars.bat || goto :eof
+      $(python.exe) -m tox -e $(tox.env)
     displayName: 'Run tests'
 
   - task: PublishTestResults@2
     inputs:
       testResultsFiles: 'build/test-results/$(tox.env).xml'
       testRunTitle: '$(tox.env)'
     condition: succeededOrFailed()
+
+  - script: call scripts\upload-coverage.bat
+    displayName: 'Report and upload coverage'
+    condition: eq(variables['PYTEST_COVERAGE'], '1')
+    env:
+      PYTHON: $(python.exe)
+      CODECOV_TOKEN: $(CODECOV_TOKEN)
+      PYTEST_CODECOV_NAME: $(tox.env)

bench/skip.py

@@ -2,7 +2,6 @@
 
 import pytest
 
-
 SKIP = True
 
 

changelog/4861.bugfix.rst

@@ -0,0 +1 @@
+Improve validation of contents written to captured output so it behaves the same as when capture is disabled.