path: root/.gitlab-ci.yml

include:
  template: License-Management.gitlab-ci.yml

stages:
- prepare
- prebuild
- test
- coverage
- build
- package
- release
- cleanup

variables:
  CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-2
  GOROOT: /usr/local/go
  # Following variables are used in some jobs to install specified software
  DOCKER_MACHINE_VERSION: "0.14.0"
  DUMB_INIT_VERSION: "1.0.2"
  GIT_VERSION: "2.21.0"
  GIT_VERSION_BUILD: "1"
  GIT_LFS_VERSION: "2.7.1"
  LICENSE_MANAGEMENT_SETUP_CMD: echo "Skip setup. Dependency already vendored"

before_script:
  - unset GPG_KEY

image: $CI_IMAGE

# Skip all jobs except the ones that begin and end with 'docs'.
# Used for commits including ONLY documentation changes.
# https://docs.gitlab.com/ce/development/writing_documentation.html#testing
.except_docs: &except_docs
  except:
  - /(^docs[\/-].*|.*-docs$)/

.windows_base:
  variables: &windows_base_variables
    PKG_NAMESPACE: gitlab.com\gitlab-org
    PKG_NAME: gitlab-runner
    GOPATH: "$CI_PROJECT_DIR\\.gopath"
    PKG_BASE_DIR: "$CI_PROJECT_DIR\\.gopath\\src\\$PKG_NAMESPACE"
    PKG_BUILD_DIR: "$CI_PROJECT_DIR\\.gopath\\src\\$PKG_NAMESPACE\\$PKG_NAME"

.docker: &docker
  <<: *except_docs
  services:
  - docker:dind
  variables: &docker_variables
    CI_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/ci:1.8.7-2
    DOCKER_DRIVER: overlay2
    DOCKER_HOST: tcp://docker:2375
  tags:
  - docker

.windows1803: &windows1803
  variables: &windows1803_variables
    <<: *windows_base_variables
    WINDOWS_VERSION: servercore1803
  tags:
  - windows-1803
  - shell

.windows1809: &windows1809
  variables: &windows1809_variables
    <<: *windows_base_variables
    WINDOWS_VERSION: servercore1809
  tags:
  - windows-1809
  - shell

#
# Prepare stage
#

.image_builder: &image_builder
  <<: *docker
  stage: prepare
  image: docker:git
  script:
  - source ./ci/build_ci_image

prepare ci image:
  <<: *image_builder
  variables:
    BUILD_IMAGE: $CI_IMAGE
    BUILD_DOCKERFILE: ./dockerfiles/ci/Dockerfile
  only:
  - /\Adocker-ci-image\/.*/

prepare alpine-no-root image:
  <<: *image_builder
  variables:
    BUILD_IMAGE: registry.gitlab.com/gitlab-org/gitlab-runner/alpine-no-root:latest
    BUILD_DOCKERFILE: ./tests/dockerfiles/alpine-no-root/Dockerfile
  only:
  - /\Adocker-alpine-no-root-image\/.*/

#
# Prebuild stage
#

helper images:
  <<: *docker
  stage: prebuild
  script:
  - make helper-build helper-docker
  artifacts:
    paths:
    - out/helper-images/
    - dockerfiles/build/binaries/*-windows.exe
    - .gopath/bin
    - .gopath/.ok
    expire_in: 7d

clone test repo:
  <<: *except_docs
  stage: prebuild
  script:
  - mkdir tmp
  - succeed=0
  - for i in {1..3}; do git clone https://gitlab.com/gitlab-org/ci-cd/tests/gitlab-test.git tmp/gitlab-test && succeed=1 && break; echo "retrying"; done
  - '[[ "$succeed" -eq 1 ]]'
  artifacts:
    paths:
    - tmp/gitlab-test
    expire_in: 7d

# Review docs base
.review-docs: &review-docs
  image: ruby:2.4-alpine
  before_script:
    - unset GPG_KEY
    - gem install gitlab --no-doc
    # We need to download the script rather than clone the repo since the
    # review-docs-cleanup job will not be able to run when the branch gets
    # deleted (when merging the MR).
    - apk add --update openssl
    - wget https://gitlab.com/gitlab-org/gitlab-ce/raw/master/scripts/trigger-build-docs
    - chmod 755 trigger-build-docs
  cache: {}
  dependencies: []
  variables:
    GIT_STRATEGY: none
  when: manual
  only:
    - branches

# Trigger a docs build in gitlab-docs
# Useful to preview the docs changes live
# https://docs.gitlab.com/ee/development/writing_documentation.html#previewing-the-changes-live
review-docs-deploy:
  <<: *review-docs
  stage: prebuild
  environment:
    name: review-docs/$CI_COMMIT_REF_SLUG
    # DOCS_REVIEW_APPS_DOMAIN and DOCS_GITLAB_REPO_SUFFIX are secret variables
    # Discussion: https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/14236/diffs#note_40140693
    url: http://$CI_ENVIRONMENT_SLUG.$DOCS_REVIEW_APPS_DOMAIN/$DOCS_GITLAB_REPO_SUFFIX
    on_stop: review-docs-cleanup
  script:
    - ./trigger-build-docs deploy

# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
  <<: *review-docs
  stage: cleanup
  environment:
    name: review-docs/$CI_COMMIT_REF_SLUG
    action: stop
  script:
    - ./trigger-build-docs cleanup

tests definitions:
  <<: *except_docs
  stage: prebuild
  # Using Go 1.10, because the `go test -list` is unavailable in <1.9
  # and it's much faster in 1.10. And since there is no real test execution
  # in this step, the version difference doesn't matter
  image: golang:1.10
  script:
  - apt-get update
  - apt-get install -y make
  - source ci/touch_make_dependencies
  - make parallel_test_prepare
  artifacts:
    paths:
    - testsdefinitions.txt
    expire_in: 7d
  tags:
  - gitlab-org

dep check:
  <<: *except_docs
  stage: prebuild
  cache: {}
  dependencies:
  script:
  - make dep_check

dep status:
  <<: *except_docs
  stage: prebuild
  cache: {}
  dependencies:
  script:
  - make dep_status
  only:
    variables:
    - $CI_DEP_STATUS_ENABLED

check feature flags docs:
  stage: prebuild
  cache: {}
  dependencies:
  script:
  - cp docs/configuration/feature-flags.md docs/configuration/feature-flags.md.orig
  - make update_feature_flags_docs
  - |
    diff docs/configuration/feature-flags.md.orig docs/configuration/feature-flags.md || (
      echo
      echo "Feature Flags list in documentation is not up-to-date"
      echo "Run 'make update_feature_flags_docs' to upate it"
      echo
      exit 1
    )

#
# Test stage
#

code_quality:
  <<: *except_docs
  stage: test
  image: docker:git
  services:
    - docker:dind
  variables:
    DOCKER_DRIVER: overlay2
    CODECLIMATE_FORMAT: json
  cache: {}
  dependencies: []
  script:
    - ./scripts/codequality analyze -f json --dev | tee gl-code-quality-report.json
  artifacts:
    paths: [gl-code-quality-report.json]
    expire_in: 7d

unit test: &unit_tests
  <<: *docker
  stage: test
  parallel: 5
  variables:
    <<: *docker_variables
    GIT_LFS_256_CHECKSUM: 7be85238cbbb957ab25de52b60279d40ba40d3faa72eeb2cb9fa77d6d92381e5
  before_script:
    # This' script content as well as the definition of GIT_LFS_256_CHECKSUM should be moved to
    # "prepare ci image". Git LFS should be installed inside of the CI image. Unfortunately, building
    # this image is currently blocked because of https://gitlab.com/gitlab-org/gitlab-runner/issues/3946.
    # After we will resolve this issue, we should update the CI configuration and move the Git LFS installation
    # to the proper place.
    - ./ci/install_git_lfs
  script:
    - source ci/touch_make_dependencies
    - make parallel_test_execute
  artifacts:
    paths:
      - .cover/*
      - .testoutput/*
    when: always
    expire_in: 7d

unit test with race:
  <<: *unit_tests
  allow_failure: true
  variables:
    <<: *docker_variables
    GIT_LFS_256_CHECKSUM: 7be85238cbbb957ab25de52b60279d40ba40d3faa72eeb2cb9fa77d6d92381e5
    CGO_ENABLED: "1"
    TESTFLAGS: "-cover -race"

docs lint:
  image: "registry.gitlab.com/gitlab-org/gitlab-build-images:gitlab-docs-lint"
  stage: test
  cache: {}
  dependencies: []
  before_script:
    - unset GPG_KEY
  script:
    - mv docs/ /tmp/gitlab-docs/content/$DOCS_GITLAB_REPO_SUFFIX
    - cd /tmp/gitlab-docs
    # Lint Markdown
    # https://github.com/markdownlint/markdownlint/blob/master/docs/RULES.md
    - bundle exec mdl content/$DOCS_GITLAB_REPO_SUFFIX/**/*.md --rules \
        MD001,MD002,MD003,MD005,MD006,MD009,MD010,MD011,MD012,MD018,MD019,MD022,\
        MD023,MD025,MD027,MD028,MD029,MD030,MD031,MD032,MD034,MD037,MD038,MD039
    # Build HTML from Markdown
    - bundle exec nanoc
    # Check the internal links
    - bundle exec nanoc check internal_links
    # Check the internal anchor links
    - bundle exec nanoc check internal_anchors

.windows test:
  extends: .except_docs
  stage: test
  parallel: 3
  variables:
    GOROOT: "C:\\Go"
  before_script:
  - if (Test-Path $GOPATH) { Remove-Item -Force -Recurse $GOPATH }
  - New-Item $PKG_BASE_DIR -ItemType directory -Force
  - New-Item -ItemType SymbolicLink -Path $PKG_BUILD_DIR -Value $CI_PROJECT_DIR
  - cd $PKG_BUILD_DIR
  script:
  - .\ci\test_windows.ps1
  after_script:
  - if (Test-Path $PKG_BUILD_DIR) { $(Get-Item $PKG_BUILD_DIR).Delete() }
  artifacts:
    paths:
    - .testoutput/*
    when: always
    expire_in: 7d
  allow_failure: true
  only:
  - /.*/@gitlab-org/gitlab-runner

windows 1803 tests:
  <<: *windows1803
  extends: .windows test

windows 1809 tests:
  <<: *windows1809
  extends: .windows test

git 1.7.1:
  <<: *except_docs
  image: centos:6
  stage: test
  variables:
    GOROOT: /usr/local/go
    GOPATH: /go
    CGO_ENABLED: 0
    OUR_PACKAGES: gitlab.com/gitlab-org/gitlab-runner/executors/shell gitlab.com/gitlab-org/gitlab-runner/shells
  before_script:
    - unset GPG_KEY
    - yum install -y -q git && git version
    - curl https://storage.googleapis.com/golang/go1.8.7.linux-amd64.tar.gz | tar zx -C $(dirname "$GOROOT")
    - export PATH=${GOROOT}/bin:$PATH
    - source ci/touch_make_dependencies
  script:
    - make test

#
# Coverage stage
#

.coverage_job: &coverage_job
  <<: *except_docs
  stage: coverage
  image: golang:1.10
  when: always

test coverage report:
  <<: *coverage_job
  coverage: /regular total:\s+\(statements\)\s+\d+.\d+\%/
  script:
    - source ci/touch_make_dependencies
    - make parallel_test_coverage_report
    - make parallel_test_coverage_report TESTFLAGS="-cover -race"
  artifacts:
    paths:
      - out/coverage/*
    expire_in: 7d

race conditions detector:
  <<: *coverage_job
  script:
  - source ci/touch_make_dependencies
  - make check_race_conditions

test junit report:
  <<: *coverage_job
  image: $CI_IMAGE
  script:
  - source ci/touch_make_dependencies
  - make parallel_test_junit_report
  artifacts:
    reports:
      junit: out/junit/report.xml

.check windows test failures:
  stage: coverage
  script:
  - ./scripts/check_windows_failures
  artifacts:
    paths:
    - .test-failures.*.txt.updated
    expire_in: 7d
  when: always

check windows 1803 test failures:
  variables:
    <<: *windows1803_variables
  extends: .check windows test failures

check windows 1809 test failures:
  variables:
    <<: *windows1809_variables
  extends: .check windows test failures

#
# Build stage
#

.binaries: &binaries
  <<: *except_docs
  stage: build
  script:
  - source ci/touch_make_dependencies
  - export platforms=$(echo $CI_JOB_NAME | sed 's|binaries ||')
  - make build BUILD_PLATFORMS="-osarch='$platforms'"
  artifacts:
    paths:
    - out/binaries/
    expire_in: 7d
  tags:
  - gitlab-org

binaries darwin/386 darwin/amd64: *binaries
binaries freebsd/386 freebsd/amd64 freebsd/arm: *binaries
binaries linux/386 linux/amd64 linux/arm: *binaries
binaries windows/386 windows/amd64: *binaries

#
# Package stage
#

.packages: &packages
  <<: *except_docs
  stage: package
  before_script:
  - |
    # checking GPG signing support
    if [ -n "$GPG_KEY" ]; then
      echo "$GPG_KEY" | gpg --batch --no-tty --allow-secret-key-import --import -
      unset GPG_KEY
      export GPG_KEYID=$(gpg --with-colon --list-secret-keys | head -n1 | cut -d : -f 5)
    else
      echo -e "\033[0;31m****** GPG signing disabled ******\033[0m"
    fi
  script:
  - source ci/touch_make_dependencies
  - make ${CI_JOB_NAME}
  artifacts:
    paths:
    - out/deb/
    - out/rpm/
    expire_in: 7d
  tags:
  - gitlab-org

package-deb: *packages
package-rpm: *packages

#
# Release stage
#

.release_docker_images: &release_docker_images
  <<: *docker
  stage: release
  variables: &release_docker_images_variables
    <<: *docker_variables
    PUBLISH_IMAGES: "true"
    PUSH_TO_DOCKER_HUB: "true"
    DOCKER_MACHINE_CHECKSUM: "a4c69bffb78d3cfe103b89dae61c3ea11cc2d1a91c4ff86e630c9ae88244db02"
    DUMB_INIT_CHECKSUM: "a8defac40aaca2ca0896c7c5adbc241af60c7c3df470c1a4c469a860bd805429"
    GIT_LFS_VERSION: "2.7.1"
    GIT_LFS_CHECKSUM: "c8952ee72af214e3669f834d829e8a0a3becd160dead18237f99e40d75a3e920"

  script:
  - source ci/touch_make_dependencies
  - make release_docker_images
  tags:
  - release

.build_windows_docker_images:
  <<: *except_docs
  stage: release
  variables:
    PUSH_TO_DOCKER_HUB: "false"
    GIT_256_CHECKSUM: "bd91db55bd95eaa80687df28877e2df8c8858a0266e9c67331cfddba2735f25c"
    GIT_LFS_256_CHECKSUM: "5c12db9728b53cba23e5f58f4a53d88cb2132e82fa1de0f8a79ce9d112e4d396"
  before_script:
  - if (Test-Path env:GPG_KEY) { Remove-Item Env:GPG_KEY }
  script:
  - .\ci\build_release_windows_images.ps1


.release: &release
  <<: *except_docs
  stage: release
  before_script:
  - unset GPG_KEY
  - source ci/touch_make_dependencies
  tags:
  - release

.release_s3: &release_s3
  <<: *release
  script:
  - make release_s3
  - |
    # publish release on gitlab.com
    if [[ -n "${CI_COMMIT_TAG}" ]]; then
      ./ci/release_gitlab
    else
      echo -e "\033[0;31m****** gitlab publishing disabled ******\033[0m"
    fi

.release_packagecloud: &release_packagecloud
  <<: *release
  script:
  - make release_packagecloud

.release_development: &release_development
  only:
  - branches@gitlab-org/gitlab-runner
  except:
  - master@gitlab-org/gitlab-runner
  - /(^docs[\/-].*|.*-docs$)/

.release_bleeding_edge: &release_bleeding_edge
  only:
  - master@gitlab-org/gitlab-runner
  - /\Av[0-9]+\.[0-9]+\.[0-9]+-rc[0-9]+\Z/@gitlab-org/gitlab-runner

.release_stable: &release_stable
  only:
  - /\Av[0-9]+\.[0-9]+\.[0-9]+\Z/@gitlab-org/gitlab-runner

development S3:
  <<: *release_s3
  <<: *release_development
  environment:
    name: development/s3/${CI_COMMIT_REF_NAME}
    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html

development docker images:
  <<: *release_docker_images
  variables:
    <<: *release_docker_images_variables
    PUBLISH_IMAGES: "false"
    PUSH_TO_DOCKER_HUB: "false"
  only:
  - branches
  except:
  - master
  - /(^docs[\/-].*|.*-docs$)/
  tags:
  - docker

development servercore1803 helper docker image:
  <<: *release_development
  <<: *windows1803
  extends: .build_windows_docker_images

development servercore1809 helper docker image:
  <<: *release_development
  <<: *windows1809
  extends: .build_windows_docker_images

bleeding edge S3:
  <<: *release_s3
  <<: *release_bleeding_edge
  environment:
    name: bleeding_edge/s3
    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html

bleeding edge packagecloud:
  <<: *release_packagecloud
  <<: *release_bleeding_edge
  environment:
    name: bleeding_edge/packagecloud
    url: https://packages.gitlab.com/runner/unstable

bleeding edge docker images:
  <<: *release_docker_images
  <<: *release_bleeding_edge
  environment:
    name: bleeding_edge/docker_images
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

bleeding edge servercore1803 docker images:
  <<: *release_bleeding_edge
  <<: *windows1803
  extends: .build_windows_docker_images
  variables:
    <<: *windows1803_variables
    PUSH_TO_DOCKER_HUB: "true"
  environment:
    name: bleeding_edge/docker_images/windows1803
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

bleeding edge servercore1809 docker images:
  <<: *release_bleeding_edge
  <<: *windows1809
  extends: .build_windows_docker_images
  variables:
    <<: *windows1809_variables
    PUSH_TO_DOCKER_HUB: "true"
  environment:
    name: bleeding_edge/docker_images/windows1809
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

stable S3:
  <<: *release_s3
  <<: *release_stable
  environment:
    name: stable/s3
    url: https://gitlab-runner-downloads.s3.amazonaws.com/${CI_COMMIT_REF_NAME}/index.html

stable packagecloud:
  <<: *release_packagecloud
  <<: *release_stable
  environment:
    name: stable/packagecloud
    url: https://packages.gitlab.com/runner/gitlab-runner

stable docker images:
  <<: *release_docker_images
  <<: *release_stable
  environment:
    name: stable/docker_images
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

stable servercore1803 docker images:
  <<: *release_stable
  <<: *windows1803
  extends: .build_windows_docker_images
  variables:
    <<: *windows1803_variables
    PUSH_TO_DOCKER_HUB: "true"
  environment:
    name: stable/docker_images/windows1803
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

stable servercore1809 docker images:
  <<: *release_stable
  <<: *windows1809
  extends: .build_windows_docker_images
  variables:
    <<: *windows1809_variables
    PUSH_TO_DOCKER_HUB: "true"
  environment:
    name: stable/docker_images/windows1809
    url: https://hub.docker.com/r/gitlab/gitlab-runner/tags/

static QA:
  <<: *except_docs
  stage: release
  image: alpine:3.7
  cache: {}
  dependencies:
  - code_quality
  script: |
    if [ "$(cat gl-code-quality-report.json)" != "[]" ] ; then
      apk add -U --no-cache jq > /dev/null
      jq -C . gl-code-quality-report.json
      exit 1
    fi