Merge branch 'track-windows-test-failures' into 'master'

Track Windows tests failures

Closes #4305

See merge request gitlab-org/gitlab-runner!1450

6 files changed, 341 insertions, 1 deletions

diff --git a/.gitignore b/.gitignore
index 181ec64c..df5df263 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,6 +24,8 @@ testsdefinitions.txt
 /.vagrant/
 
 codeclimate.json
+ci/.test-failures.servercore1803.txt.updated
+ci/.test-failures.servercore1809.txt.updated
 
 # Ignore Visual Studio Code internals
 /.vscode
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ac75d29f..b43bc571 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -315,6 +315,11 @@ docs lint:
   - .\ci\test_windows.ps1
   after_script:
   - if (Test-Path $PKG_BUILD_DIR) { $(Get-Item $PKG_BUILD_DIR).Delete() }
+  artifacts:
+    paths:
+    - .testoutput/*
+    when: always
+    expire_in: 7d
   allow_failure: true
   only:
   - /.*/@gitlab-org/gitlab-runner
@@ -383,6 +388,26 @@ test junit report:
     reports:
       junit: out/junit/report.xml
 
+.check windows test failures:
+  stage: coverage
+  script:
+  - ./scripts/check_windows_failures
+  artifacts:
+    paths:
+    - .test-failures.*.txt.updated
+    expire_in: 7d
+  when: always
+
+check windows 1803 test failures:
+  variables:
+    <<: *windows1803_variables
+  extends: .check windows test failures
+
+check windows 1809 test failures:
+  variables:
+    <<: *windows1809_variables
+  extends: .check windows test failures
+
 #
 # Build stage
 #
diff --git a/ci/.test-failures.servercore1803.txt b/ci/.test-failures.servercore1803.txt
new file mode 100644
index 00000000..c70c72bb
--- /dev/null
+++ b/ci/.test-failures.servercore1803.txt
@@ -0,0 +1,93 @@
+TestBuildCancel
+TestBuildCancel/cmd
+TestBuildCancel/powershell
+TestBuildMultilineCommand
+TestBuildWithDebugTrace
+TestBuildWithDebugTrace/cmd
+TestBuildWithGitFetchSubmoduleStrategyRecursive
+TestBuildWithGitFetchSubmoduleStrategyRecursive/cmd
+TestBuildWithGitFetchSubmoduleStrategyRecursive/powershell
+TestBuildWithGitSSLAndStrategyFetch
+TestBuildWithGitSSLAndStrategyFetch/cmd
+TestBuildWithGitSSLAndStrategyFetch/powershell
+TestBuildWithGitStrategyCloneWithLFS
+TestBuildWithGitStrategyCloneWithLFS/powershell
+TestBuildWithGitStrategyCloneWithoutLFS
+TestBuildWithGitStrategyCloneWithoutLFS/cmd
+TestBuildWithGitStrategyCloneWithoutLFS/powershell
+TestBuildWithGitStrategyCloneWithUserDisabledLFS
+TestBuildWithGitStrategyCloneWithUserDisabledLFS/cmd
+TestBuildWithGitStrategyCloneWithUserDisabledLFS/powershell
+TestBuildWithGitStrategyFetchWithLFS
+TestBuildWithGitStrategyFetchWithLFS/cmd
+TestBuildWithGitStrategyFetchWithLFS/powershell
+TestBuildWithGitStrategyFetchWithUserDisabledLFS
+TestBuildWithGitStrategyFetchWithUserDisabledLFS/cmd
+TestBuildWithGitStrategyFetchWithUserDisabledLFS/powershell
+TestBuildWithGitSubmoduleModified
+TestBuildWithGitSubmoduleModified/cmd
+TestBuildWithGitSubmoduleModified/powershell
+TestBuildWithGitSubmoduleStrategyNormal
+TestBuildWithGitSubmoduleStrategyNormal/cmd
+TestBuildWithGitSubmoduleStrategyNormal/powershell
+TestBuildWithGitSubmoduleStrategyRecursive
+TestBuildWithGitSubmoduleStrategyRecursive/cmd
+TestBuildWithGitSubmoduleStrategyRecursive/powershell
+TestBuildWithGoodGitSSLCAInfo
+TestBuildWithGoodGitSSLCAInfo/cmd
+TestBuildWithGoodGitSSLCAInfo/powershell
+TestBuildWithIndexLock
+TestBuildWithIndexLock/cmd
+TestBuildWithIndexLock/powershell
+TestBuildWithSubmoduleLFSDisabledSmudging
+TestBuildWithSubmoduleLFSDisabledSmudging/cmd
+TestBuildWithSubmoduleLFSDisabledSmudging/powershell
+TestBuildWithSubmoduleLFSPullsLFSObject
+TestBuildWithSubmoduleLFSPullsLFSObject/cmd
+TestBuildWithSubmoduleLFSPullsLFSObject/powershell
+TestCacheInContainer
+TestDefaultVariables
+TestDefaultVariables/get_default_CI_PROJECT_DIR_value
+TestDefaultVariables/get_overwritten_CI_PROJECT_DIR_value
+TestDockerCommandBuildAbort
+TestDockerCommandBuildCancel
+TestDockerCommandBuildFail
+TestDockerCommandDisableEntrypointOverwrite
+TestDockerCommandDisableEntrypointOverwrite/Disabled_-_no_services
+TestDockerCommandDisableEntrypointOverwrite/Disabled_-_services
+TestDockerCommandDisableEntrypointOverwrite/Enabled_-_no_services
+TestDockerCommandDisableEntrypointOverwrite/Enabled_-_services
+TestDockerCommandMissingImage
+TestDockerCommandMissingTag
+TestDockerCommandNoRootImage
+TestDockerCommandOutput
+TestDockerCommandSuccessRun
+TestDockerCommandTwoServicesFromOneImage
+TestDockerCommandUsingBuildsVolume
+TestDockerCommandUsingBuildsVolume/disables_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingBuildsVolume/enables_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingBuildsVolume/uses_default_state_of_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingCustomClonePath
+TestDockerCommandUsingCustomClonePath/uses_custom_clone_path
+TestDockerCommandWithAllowedImagesRun
+TestDockerCommandWithBrokenGitSSLCAInfo
+TestDockerCommandWithDoingPruneAndAfterScript
+TestDockerCommandWithGitSSLCAInfo
+TestDockerCommandWithHelperImageConfig
+TestDockerExtendedConfigurationFromJob
+TestDockerExtendedConfigurationFromJob/example-0
+TestDockerExtendedConfigurationFromJob/example-1
+TestDockerImageNameFromVariable
+TestDockerPrivilegedServiceAccessingBuildsFolder
+TestDockerServiceNameFromVariable
+TestDockerWatchOn_1_12_4
+TestInteractiveTerminal
+TestList
+TestList/machines_directory_is_invalid
+TestMachineIdleLimits
+TestMachineProviderDeadInterval
+TestNewDockerMachineCommandCtx
+TestNewDockerMachineCommandCtx/MACHINE_BUGSNAG_API_TOKEN_is_defined_by_the_user
+TestNewDockerMachineCommandCtx/MACHINE_BUGSNAG_API_TOKEN_is_not_defined_by_the_user
+TestStartBuild
+TestStartBuild/valid_GIT_CLONE_PATH_using_CI_BUILDS_DIR_was_specified
diff --git a/ci/.test-failures.servercore1809.txt b/ci/.test-failures.servercore1809.txt
new file mode 100644
index 00000000..c70c72bb
--- /dev/null
+++ b/ci/.test-failures.servercore1809.txt
@@ -0,0 +1,93 @@
+TestBuildCancel
+TestBuildCancel/cmd
+TestBuildCancel/powershell
+TestBuildMultilineCommand
+TestBuildWithDebugTrace
+TestBuildWithDebugTrace/cmd
+TestBuildWithGitFetchSubmoduleStrategyRecursive
+TestBuildWithGitFetchSubmoduleStrategyRecursive/cmd
+TestBuildWithGitFetchSubmoduleStrategyRecursive/powershell
+TestBuildWithGitSSLAndStrategyFetch
+TestBuildWithGitSSLAndStrategyFetch/cmd
+TestBuildWithGitSSLAndStrategyFetch/powershell
+TestBuildWithGitStrategyCloneWithLFS
+TestBuildWithGitStrategyCloneWithLFS/powershell
+TestBuildWithGitStrategyCloneWithoutLFS
+TestBuildWithGitStrategyCloneWithoutLFS/cmd
+TestBuildWithGitStrategyCloneWithoutLFS/powershell
+TestBuildWithGitStrategyCloneWithUserDisabledLFS
+TestBuildWithGitStrategyCloneWithUserDisabledLFS/cmd
+TestBuildWithGitStrategyCloneWithUserDisabledLFS/powershell
+TestBuildWithGitStrategyFetchWithLFS
+TestBuildWithGitStrategyFetchWithLFS/cmd
+TestBuildWithGitStrategyFetchWithLFS/powershell
+TestBuildWithGitStrategyFetchWithUserDisabledLFS
+TestBuildWithGitStrategyFetchWithUserDisabledLFS/cmd
+TestBuildWithGitStrategyFetchWithUserDisabledLFS/powershell
+TestBuildWithGitSubmoduleModified
+TestBuildWithGitSubmoduleModified/cmd
+TestBuildWithGitSubmoduleModified/powershell
+TestBuildWithGitSubmoduleStrategyNormal
+TestBuildWithGitSubmoduleStrategyNormal/cmd
+TestBuildWithGitSubmoduleStrategyNormal/powershell
+TestBuildWithGitSubmoduleStrategyRecursive
+TestBuildWithGitSubmoduleStrategyRecursive/cmd
+TestBuildWithGitSubmoduleStrategyRecursive/powershell
+TestBuildWithGoodGitSSLCAInfo
+TestBuildWithGoodGitSSLCAInfo/cmd
+TestBuildWithGoodGitSSLCAInfo/powershell
+TestBuildWithIndexLock
+TestBuildWithIndexLock/cmd
+TestBuildWithIndexLock/powershell
+TestBuildWithSubmoduleLFSDisabledSmudging
+TestBuildWithSubmoduleLFSDisabledSmudging/cmd
+TestBuildWithSubmoduleLFSDisabledSmudging/powershell
+TestBuildWithSubmoduleLFSPullsLFSObject
+TestBuildWithSubmoduleLFSPullsLFSObject/cmd
+TestBuildWithSubmoduleLFSPullsLFSObject/powershell
+TestCacheInContainer
+TestDefaultVariables
+TestDefaultVariables/get_default_CI_PROJECT_DIR_value
+TestDefaultVariables/get_overwritten_CI_PROJECT_DIR_value
+TestDockerCommandBuildAbort
+TestDockerCommandBuildCancel
+TestDockerCommandBuildFail
+TestDockerCommandDisableEntrypointOverwrite
+TestDockerCommandDisableEntrypointOverwrite/Disabled_-_no_services
+TestDockerCommandDisableEntrypointOverwrite/Disabled_-_services
+TestDockerCommandDisableEntrypointOverwrite/Enabled_-_no_services
+TestDockerCommandDisableEntrypointOverwrite/Enabled_-_services
+TestDockerCommandMissingImage
+TestDockerCommandMissingTag
+TestDockerCommandNoRootImage
+TestDockerCommandOutput
+TestDockerCommandSuccessRun
+TestDockerCommandTwoServicesFromOneImage
+TestDockerCommandUsingBuildsVolume
+TestDockerCommandUsingBuildsVolume/disables_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingBuildsVolume/enables_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingBuildsVolume/uses_default_state_of_FF_USE_LEGACY_BUILDS_DIR_FOR_DOCKER
+TestDockerCommandUsingCustomClonePath
+TestDockerCommandUsingCustomClonePath/uses_custom_clone_path
+TestDockerCommandWithAllowedImagesRun
+TestDockerCommandWithBrokenGitSSLCAInfo
+TestDockerCommandWithDoingPruneAndAfterScript
+TestDockerCommandWithGitSSLCAInfo
+TestDockerCommandWithHelperImageConfig
+TestDockerExtendedConfigurationFromJob
+TestDockerExtendedConfigurationFromJob/example-0
+TestDockerExtendedConfigurationFromJob/example-1
+TestDockerImageNameFromVariable
+TestDockerPrivilegedServiceAccessingBuildsFolder
+TestDockerServiceNameFromVariable
+TestDockerWatchOn_1_12_4
+TestInteractiveTerminal
+TestList
+TestList/machines_directory_is_invalid
+TestMachineIdleLimits
+TestMachineProviderDeadInterval
+TestNewDockerMachineCommandCtx
+TestNewDockerMachineCommandCtx/MACHINE_BUGSNAG_API_TOKEN_is_defined_by_the_user
+TestNewDockerMachineCommandCtx/MACHINE_BUGSNAG_API_TOKEN_is_not_defined_by_the_user
+TestStartBuild
+TestStartBuild/valid_GIT_CLONE_PATH_using_CI_BUILDS_DIR_was_specified
diff --git a/ci/test_windows.ps1 b/ci/test_windows.ps1
index 9b5e3d99..e786995d 100644
--- a/ci/test_windows.ps1
+++ b/ci/test_windows.ps1
@@ -20,13 +20,16 @@ Write-Information "Suite index: $env:CI_NODE_INDEX"
 Write-Information "Execution size: $executionSize"
 Write-Information "Execution offset: $executionOffset"
 
+New-Item -ItemType "directory" -Path ".\" -Name ".testoutput"
+
 $failed = @()
 Get-Content $testsDefinitionsFile | Select-Object -skip $executionOffset -first $executionSize | ForEach-Object {
     $pkg, $index, $tests = $_.Split(" ", 3)
+    $pkgSlug = ((echo $pkg | % { $_ -replace "[^a-z0-9_]","_" }))
 
     Write-Information "`r`n`r`n--- Starting part $index of go tests of '$pkg' package:`r`n`r`n"
 
-    go test -v $pkg -run "$tests"
+    go test -v $pkg -run "$tests" | Tee ".testoutput/${pkgSlug}.${index}.windows.${WINDOWS_VERSION}.output.txt"
 
     if ($LASTEXITCODE -ne 0) {
         $failed += "$pkg-$index"
diff --git a/scripts/check_windows_failures b/scripts/check_windows_failures
new file mode 100755
index 00000000..40079fd9
--- /dev/null
+++ b/scripts/check_windows_failures
@@ -0,0 +1,124 @@
+#!/usr/bin/env bash
+
+set -eo pipefail
+
+borderTop() {
+    echo
+    echo "========================================================================================================================="
+}
+
+borderBottom() {
+    echo "========================================================================================================================="
+    echo
+}
+
+WINDOWS_VERSION=${WINDOWS_VERSION:-servercore1809}
+
+testOutputDir="./.testoutput"
+windowsTestOutputPattern="${testOutputDir}/*.windows.${WINDOWS_VERSION}.output.txt"
+
+trackedFailuresFile="./ci/.test-failures.${WINDOWS_VERSION}.txt"
+if [[ ! -f "${trackedFailuresFile}" ]]; then
+    touch "${trackedFailuresFile}"
+fi
+
+trackedFailuresCount=$(wc -l < "${trackedFailuresFile}")
+
+existingFailures=()
+newFailures=()
+
+for file in ${windowsTestOutputPattern}; do
+    for failure in $(iconv -f utf-16 -t utf-8 "$file" | grep -Eo "\\--- FAIL: [^ ]+" | awk '{print $3}'); do
+        if grep "^${failure}$" "${trackedFailuresFile}" >/dev/null 2>&1; then
+            existingFailures+=("${failure}")
+        else
+            newFailures+=("${failure}")
+        fi
+    done
+done
+
+existingFailuresCount=${#existingFailures[@]}
+newFailuresCount=${#newFailures[@]}
+
+fixedFailuresCount=$((trackedFailuresCount - existingFailuresCount))
+
+borderTop
+echo "    Tracked failures: ${trackedFailuresCount}"
+echo "   Existing failures: ${existingFailuresCount}"
+echo "        New failures: ${newFailuresCount}"
+echo "  Fixed (?) failures: ${fixedFailuresCount}"
+borderBottom
+
+if [[ ${fixedFailuresCount} -gt -0 ]]; then
+    tmpFile="$(mktemp)"
+    updatedTrackedFailuresFile="${trackedFailuresFile}.updated"
+
+    touch "${updatedTrackedFailuresFile}"
+
+    cp "${trackedFailuresFile}" "${tmpFile}"
+
+    for failure in "${existingFailures[@]}"; do
+        echo "${failure}" >> "${updatedTrackedFailuresFile}"
+
+        grep -v "^${failure}$" "${tmpFile}" > "${tmpFile}.new"
+        mv "${tmpFile}.new" "${tmpFile}"
+    done
+
+    borderTop
+    echo "  Tests that were probably fixed:"
+    echo
+
+    while read -r file; do
+        echo "  - ${file}"
+    done < "${tmpFile}"
+
+    echo
+    echo "  Please consider updating the tracked failures file"
+    echo "  Updated version saved as artifact: ${updatedTrackedFailuresFile}"
+    borderBottom
+
+    rm "${tmpFile}"
+fi
+
+if [[ ${newFailuresCount} -gt 0 ]]; then
+    borderTop
+    echo "  Windows tests created new failures:"
+    echo
+
+    for failure in "${newFailures[@]}"; do
+        echo "  - ${failure}"
+    done
+
+    echo
+    echo "  This is unacceptable! Fix the failures!"
+    echo "  This job will not pass untill it's done!"
+    borderBottom
+
+    exit 1
+fi
+
+if [[ ${existingFailuresCount} -eq 0 ]]; then
+    borderTop
+    echo "  All tests are fixed! Good job!"
+    echo "  You can now update .gitlab-ci.yml file and remove this CI job."
+    borderBottom
+
+    exit 0
+fi
+
+if [[ ${fixedFailuresCount} -gt 0 ]]; then
+    borderTop
+    echo "  It looks like you've fixed some Windows failures (probably)."
+    echo "  KEEP GOING!"
+    borderBottom
+
+    exit 0
+fi
+
+borderTop
+echo "  It's not worse. But it could be better!"
+echo "  If you want to have some fun, chose one of the Windows"
+echo "  test failures and try to fix it!"
+echo "  And remember: Developers! Developers! Developers!"
+borderBottom
+