拓扑#

我们假设先将它分为四部分

devops-da-2

YAML#

文件如下:

stages:
  - code-check
  - deploy-test
  - dast
  - deploy-production
  - performance
# 变量
variables:
  WWWDIR: "/data/wwwroot"
  ANSIBLEGROUP: "linuxea"
  LANG: "en_US.UTF-8"
  VERSION: "1.11"
  SONAR_URL: "http://10.0.1.61:9000"
  SONAR_KN: "sonar-linuxea"
  SONAR_CFG: "sonar-scanner -Dsonar.host.url=$SONAR_URL -Dsonar.projectKey=$SONAR_KN -Dsonar.projectName=$SONAR_KN -Dsonar.projectBaseDir=/root/src -Dsonar.sources=./ -Dsonar.java.binaries=./"
  website: "http://zabbix.dtops.cc"
# 所有 stage 之前的操作
before_script:
  - echo "start now!"
  - export LANG=en_US.UTF-8
  - export LANGUAGE=en_US:en
  - export LC_ALL=en_US.UTF-8 
  - sudo chown -R gitlab-runner.gitlab-runner ${PWD} || true
# &job_docker_group
.code_check_Static: &job_docker_group
  stage: code-check
  image: docker:stable
  variables:
    DOCKER_DRIVER: overlay2
    PATHD: /home/gitlab-runner/Increment/
  allow_failure: true
  services:
    - docker:stable-dind 
# &zap_docker_group
.zap_check: &zap_docker_group
  stage: dast
  image: registry.gitlab.com/gitlab-org/security-products/zaproxy
  variables:
    login_url: "https://example.com/sign-in"
    john_name: "USERNAME"
    john_pwd: "PASSWORD"
  allow_failure: true   
  services:
    - docker:stable-dind   
.performance: &performance
  stage: performance
  image: docker:git
  variables:
    URL: $website
  services:
    - docker:stable-dind   
# 增量生产部署    
.production: &production   
  stage: deploy-production
  environment:
    name: production
    url: https://www.linuxea.com
  only:
    - master      
# sonarqube代码质量检查
1/8 sonarqube:
  stage: code-check
  script: 
    - docker run --rm
      --volume  $(pwd):/root/src 
      --volume /etc/localtime:/etc/localtime:ro
      --volume /var/run/docker.sock:/var/run/docker.sock
      "newtmitch/sonar-scanner" $SONAR_CFG  || true
    - sudo chown -R gitlab-runner.gitlab-runner ${PWD}
    - date
  artifacts:
    paths: [sonarqube.json]
# 代码全量静态质量检查
2/8 code_quality:
  <<: *job_docker_group
  script:
    - if [ `docker ps -a|egrep "codeclimate|gitlab"|wc -l` -gt 0 ];then  docker ps -a|docker rm -f $(egrep "gitlab|codeclimate"|awk -F' ' 'END{print $NF}'); else echo "Nothing Codeclimate Runing"; fi
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run --rm
        --env SOURCE_CODE="$PWD"
        --volume "$PWD":/code
        --volume /var/run/docker.sock:/var/run/docker.sock
        --volume /etc/localtime:/etc/localtime:ro
        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code
    - date
  artifacts:
    paths: [codeclimate.json]
# 代码增量静态质量检查    
2/8 code_quality_Increment:
  <<: *job_docker_group
  script:
    - if [ `docker ps -a|egrep "codeclimate|gitlab"|wc -l` -gt 0 ];then  docker ps -a|docker rm -f $(egrep "gitlab|codeclimate"|awk -F' ' 'END{print $NF}'); else echo "Nothing Codeclimate Runing"; fi
    - export VERSION_PATH=$(echo "$CI_REPOSITORY_URL"|awk -F'/' '{print $NF}')
    - export PATH_LAST=$PATHD$VERSION_PATH
    - if [ -d $PATH_LAST ];then echo "$PATH_LAST is runing"; else mkdir $PATH_LAST -p; fi
    - git archive -o $PWD/version-$(git rev-parse HEAD)-latest.tar.gz $(git rev-parse HEAD) $(git diff --name-only HEAD HEAD~1)
    - tar xf version-$(git rev-parse HEAD)-latest.tar.gz -C $PATH_LAST
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run --rm
        --env SOURCE_CODE="$PATH_LAST"
        --volume $PATH_LAST:/code
        --volume /etc/localtime:/etc/localtime:ro
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/codequality:$SP_VERSION" /code  
    - date
  artifacts:
    paths: [codeclimate.json]
  when: on_failure
# 静态代码安全扫描    
# Static Application Security Testing
3/8 SAST:
  <<: *job_docker_group    
  script:
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run --rm
       --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}"
       --volume "$PWD":/code
       --volume /etc/localtime:/etc/localtime:ro
       --volume /var/run/docker.sock:/var/run/docker.sock
       "registry.gitlab.com/gitlab-org/security-products/sast:${SP_VERSION}" /app/bin/run /code
    - date
  artifacts:
    paths: [gl-sast-report.json]    
# 依赖扫描    
4/8 dependency-scanning:
  <<: *job_docker_group
  script:
    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run --rm
        --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}"
        --volume "$PWD:/code"
        --volume /etc/localtime:/etc/localtime:ro
        --volume /var/run/docker.sock:/var/run/docker.sock
        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
    - date
  artifacts:
    paths: [gl-dependency-scanning-report.json]    
  except:
    variables:
      - $CONTAINER_SCANNING_DISABLE

5/8 license_management:
  <<: *job_docker_group
  script:
    - if [ `docker ps -a|egrep "dependency-scanning"|wc -l` -gt 0 ];then  docker ps -a|docker rm -f $(egrep "dependency-scanning"|awk -F' ' 'END{print $NF}'); else echo "Nothing dependency-scanning Runing"; fi
    - export LICENSE_MANAGEMENT_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
    - docker run --rm
        --volume /etc/localtime:/etc/localtime:ro    
        --volume "$PWD:/code"
        "registry.gitlab.com/gitlab-org/security-products/license-management:latest" analyze /code 
#        "registry.gitlab.com/gitlab-org/security-products/license-management:$LICENSE_MANAGEMENT_VERSION" analyze /code        
    - date
  artifacts:
    paths: [gl-license-management-report.json]  
  except:
    variables:
      - $CONTAINER_SCANNING_DISABLE


# 安全测试
# Dynamic Application Security Testing
1/2 DAST_ZAP_JSON:
  <<: *zap_docker_group 
  script:
    - if [ `docker ps -a|egrep "owasp"|wc -l` -gt 0 ];then  echo "this $(docker ps -a|awk '/owasp/{print $2}') been deleted" && docker ps -a|docker rm -f $(egrep "owasp"|awk -F' ' 'END{print $NF}'); else echo "Nothing owasp/.... Runing"; fi    
    - docker run --rm
      --volume /etc/localtime:/etc/localtime:ro    
      --volume $PWD:/zap/wrk/:rw -t "registry.gitlab.com/gitlab-org/security-products/zaproxy" zap-baseline.py -t $website -g gen.conf -J gl-dast-report.json || true    
    - date      
  dependencies:
    -  deploy
  artifacts:
    paths: [gl-dast-report.json]   
#      --volume $PWD:/zap/wrk/:rw -t "owasp/zap2docker-weekly" zap-baseline.py -t $website -g gen.conf -J gl-dast-report.json || true   
2/2 DAST_ZAP_HTML:
  <<: *zap_docker_group
  script:   
    - if [ `docker ps -a|egrep "owasp"|wc -l` -gt 0 ];then  echo "this $(docker ps -a|awk '/owasp/{print $2}') been deleted" && docker ps -a|docker rm -f $(egrep "owasp"|awk -F' ' 'END{print $NF}'); else echo "Nothing owasp/.... Runing"; fi   
    - docker run --rm
     --volume /etc/localtime:/etc/localtime:ro    
     --volume $PWD:/zap/wrk/:rw -t "registry.gitlab.com/gitlab-org/security-products/zaproxy" zap-baseline.py -t $website -g gen.conf -r testreport.html  || true     
    - date     
  artifacts:
    paths: [testreport.html]     
#     --volume $PWD:/zap/wrk/:rw -t "owasp/zap2docker-weekly" zap-baseline.py -t $website -g gen.conf -r testreport.html  || true    
#    - docker run
#      --volume $PWD:/zap/wrk/    
#      --volume /etc/localtime:/etc/localtime:ro     
#      "owasp/zap2docker-stable" zap-baseline.py -t $website -g gen.conf -r testreport.html      
#部署 
deploy:
  stage: deploy-test
  environment:
    name: staging
    url: https://staging.example.com  
  only:
    - master 
  script:
    - ansible ${ANSIBLEGROUP} -m shell -a "[ -d ${WWWDIR} ]||mkdir ${WWWDIR} -p"
    - ansible ${ANSIBLEGROUP} -m synchronize -a 'src=./ dest=${WWWDIR}/ rsync_opts=--delete,--exclude=*.json,--exclude=*.yml,--exclude=*.git'
    #- ansible ${ANSIBLEGROUP} -m shell -a "chown -R 400.400 /data/wwwroot/"
  when: always  

production 10%:
  <<: *production    
  script: echo "deploy-production"
  when: manual
  allow_failure: false    
production 50%:
  <<: *production   
  script: echo "deploy-production"
  when: manual
  allow_failure: false  

production 100%:
  <<: *production 
  script: echo "deploy-production"
  when: manual
  allow_failure: false    

Sitespeed.io:
  <<: *performance
  script:
    - mkdir gitlab-exporter
    - wget -O ./gitlab-exporter/index.js https://gitlab.com/gitlab-org/gl-performance/raw/master/index.js
    - mkdir sitespeed-results
    - docker run --shm-size=1g --rm -v "$(pwd)":/sitespeed.io sitespeedio/sitespeed.io:7.1.3 --plugins.add ./gitlab-exporter --outputFolder sitespeed-results $website || true
    - cp sitespeed-results/data/performance.json $PWD/performance.json
    - sudo chown -R gitlab-runner.gitlab-runner $PWD
  artifacts:
    paths:
    - performance.json
    - sitespeed-results/ 
  when: always