Python: requests, JFrog Artifactory и деплой

Пример скрипта деплоя.

Задача скрипта — сохранить результаты билда в Artifactory и деплоить из Artifactory на хост с JBoss.

Из «особенностей» — результатом билда являются два *.ear архива разных типов:

• ${GIT_COMMIT}-${VERSION}-unlock-service.ear — бекенд из одного репозитория;
• ${GIT_COMMIT}-${VERSION}-unlock-ui.ear — фронтенд из второго репозитория.

${GIT_COMMIT} грепается из git log, ${VERSION} — из файла build.gradle, в котором девелоперы устанавливают версию.

В Artifactory они сохраняются в два разных каталога.

При деплое из Artifactory на хост — по умолчанию выбирается последний модифицированный артефакт из репозитория (?lastModified) — но можно добавить переменную $DEPLOY_VERSION в билд Jenkins-а и указать имя артефакта для деплоя.

Собственно скрипт:

#!/usr/bin/env python

from __future__ import with_statement

import os
import subprocess
import re
import requests
import argparse
import json

from fabric.api import run, env, put, sudo, hide

# hate globals
jfrog_url = 'https://project.artifactoryonline.com/project/'
jfrog_api = 'api/storage'
jfrog_repo = 'project-releases-local'
jfrog_dir = 'repo'

jfrog_user = 'username'
jfrog_pswd = 'password'

def getopts():

    # e.g. /var/lib/jenkins/jobs/Project.UI.Build/workspace/
    jenkins_home = os.environ['WORKSPACE']

    parser = argparse.ArgumentParser()

    parser.add_argument('-t',
                        '--type',
                        action='store',
                        help='"UI" or "Service" to deploy to Artifactory or JBoss instance')

    parser.add_argument('-a',
                        '--artifactory',
                        action='store_true',
                        help='Deploy build results to Artifactory')

    parser.add_argument('-j',
                        '--jboss',
                        action='store_true',
                        help='Deploy build results to JBoss host')

    parser.add_argument('-H',
                        '--host',
                        action='store',
                        help='JBoss host to deploy to.')

    parser.add_argument('-T',
                        '--test',
                        action='store_true',
                        help='Test functions option. Use it to run new functionality.')

    res = parser.parse_args()

    if res.type == 'UI':
        if res.artifactory:
            artifact_path = os.path.join(jenkins_home, 'unlock-ear', 'build', 'libs')
            artifact_to_art = versioning(res.type, artifact_path) + 'unlock-ui.ear'
        jfrog_repo_type = 'ui'
    elif res.type == 'Service':
        if res.artifactory:
            artifact_path = os.path.join(jenkins_home, 'unlock-service-ear', 'build', 'libs')
            artifact_to_art = versioning(res.type, artifact_path) + 'unlock-service.ear'
        jfrog_repo_type = 'service'
    else:
        print ('ERROR: Unknown deploy type. Please set "UI" or "Service". Exit.')
        exit(1)

    if res.artifactory:
        deploy_to_art(res.type, jfrog_repo_type, artifact_path, artifact_to_art)
    if res.jboss:
        deploy_to_env(res.type, res.host, jfrog_repo_type)
    if res.test:
        getlast(res.type, jfrog_repo_type)


def versioning(res_type, artifact_path):

    # "1.0.0-SNAPSHOT" will be parsed from here
    #
    if res_type == 'UI':
        gradlefile = 'build.gradle'
        to_grab = 'version '
    elif res_type == 'Service':
        gradlefile = os.path.join('unlock-service-ear', 'build.gradle')
        to_grab = 'version = '
    else:
        print('ERROR: unknown res_type! Exit.n')
        exit(1)

    print('Gradlefile: {}nTo Grab: {}n'.format(gradlefile, to_grab))

    if os.path.isdir('.git'):
        proc = subprocess.Popen('git log | head -n 1', shell=True, stdout=subprocess.PIPE)
        commitID = re.sub('commit ', '', proc.stdout.read())[0:7]
    else:
        print('ERROR: No .git dir found. Exit.n')
        exit(1)

    try:
        with open(gradlefile, 'r') as versionfile:
            for line in versionfile:
                if re.search('version', line):
                    # grab 1.0.0-SNAPSHOT from build.gradle file
                    version = re.sub(''', '', re.sub(to_grab, '', line)).rstrip()
                    print('VERSION: {}'.format(version))
                    # to grab only first occurrence
                    break
    except IOError:
        print('ERROR: no {} file found. Exit.n'.format(gradlefile))
        exit(1)

    arch = (commitID + '-' + version + '-')

    return arch


def deploy_to_art(res_type, jfrog_repo_type, artifact_path, artifact_to_art):

    # i.e. URL:
    # https://project.artifactoryonline.com/project/project-releases-local/repo/unlock-ui-0.0.1-SNAPSHOT.ear
    # for artifact:
    # unlock-ui-0.0.1-SNAPSHOT.ear

    deployurl = os.path.join(jfrog_url, jfrog_repo, jfrog_dir, jfrog_repo_type, artifact_to_art)

    if os.path.isfile(os.path.join(artifact_path, artifact_to_art)):

        print ('n{} artifact found in {}'.format(res_type, artifact_to_art))
        print ('nExecuting deploy {} to {}.'.format(artifact_to_art, deployurl))

        os.chdir(artifact_path)
        print ('nWorking now in {}.nnArtifacts ready to deployment:n '.format(artifact_path))
        content = os.listdir('.')
        for f in content:
            print ('{}'.format(f))

        with open(artifact_to_art, 'rb') as inartifact:
            data_put = requests.put(deployurl, auth=(jfrog_user, jfrog_pswd), data=inartifact)
            data_put.raise_for_status()
            print ('nResult code: {}'.format(data_put.status_code))
            print ('n{}n'.format(data_put.text))

    else:
        print ('nERROR: {} artifact not found in {}! Exit.n'.format(res_type, artifact_to_art))
        exit(1)


def getlast(res_type, jfrog_repo_type):

    try:
        custom_artifact_name = os.environ['DEPLOY_VERSION']
        custom_artifact_url = os.path.join(jfrog_url, jfrog_repo, jfrog_dir, jfrog_repo_type, custom_artifact_name)

        print('nATTENTION: Custom artifact name used for deploy: {}n'.format(custom_artifact_name))

        return [custom_artifact_url, custom_artifact_name]

    except KeyError:
        artifact_url = os.path.join(jfrog_url, jfrog_api, jfrog_repo, jfrog_dir, jfrog_repo_type + '?lastModified')
        data_last = requests.get(artifact_url, auth=(jfrog_user, jfrog_pswd))
        json_data = json.loads(data_last.text)

        last_artifact_api_url = json_data['uri']

        # last_artifact_api_url return URL to artifact JSON data
        # parse downloadUri from here
        geturl = requests.get(last_artifact_api_url, auth=(jfrog_user, jfrog_pswd))
        urldata = json.loads(geturl.text)
        last_artifact_url = urldata['downloadUri']

        # i.e.a1ea78b-1.0.0-SNAPSHOT-unlock-ui.ear
        last_artifact_name = json_data['uri'].rsplit('/')[-1]

        # return value with full URL to latest modified artifact in selected directory
        # from jfrog_repo_type variable - "repo/ui" or "repo/service"
        # URL example
        # https://project.artifactoryonline.com/project/api/storage/project-releases-local/repo/ui/a1ea78b-1.0.0-SNAPSHOT-unlock-ui.ear
        return [last_artifact_url, last_artifact_name]


def deploy_to_env(res_type, host, jfrog_repo_type):

    print ('nDeploying to {}'.format(host))

    env.host_string = host
    env.key_filename = [os.path.join('.ssh', 'id_rsa')]
    env.user = 'knife'
    env.project_root = os.path.join(os.sep, 'opt', 'jboss', 'standalone', 'deployments')

    artifact_to_art = getlast(res_type, jfrog_repo_type)[1]

    # Commands list

    if res_type == 'UI':
        cleanup_file_mask = '*-unlock-ui.ear'
    elif res_type == 'Service':
        cleanup_file_mask = '*-unlock-service.ear'

    # 95c25ba-1.0.0-SNAPSHOT-unlock-service.ear
    ##cleanup = ('if test -e {0}; then '
    ##           'rm -v {0};'
    ##           'else echo -e "No file found {0}, skipping.";'
    ##           'fi'.format(os.path.join(env.project_root, artifact_to_art)))

    cleanup = ('if test -e {0};'
               'then rm -v {0};'
               'else echo -e "No file found {0}, skipping.";'
               'fi'.format(os.path.join(env.project_root, cleanup_file_mask)))

    # {0} /opt/jboss/standalone/deployments
    # {1} jfrog_user
    # {2} jfrog_pswd
    # {3} full URL to artifact
    upload = ('cd {0} && wget --user {1} --password={2} {3}'.format(
        os.path.join(env.project_root),
        jfrog_user, jfrog_pswd,
        # os.path.join(jfrog_url, jfrog_repo, jfrog_dir, artifact_to_art),
        getlast(res_type, jfrog_repo_type)[0]
    ))

    chown = ('chown jboss:jboss {}'.format(os.path.join(env.project_root, artifact_to_art)))
    getcontent = ('ls -l {}'.format(os.path.join(env.project_root, artifact_to_art)))

    with hide('output', 'running'):

        print ('nRemoving outdated data:n')
        removed = sudo(cleanup)
        print ('{}n'.format(removed))

        print ('Uploading {} to {} in to {}...n'.format(os.path.join(jfrog_url, jfrog_repo, jfrog_dir, artifact_to_art),
                                                       host,
                                                       os.path.join(env.project_root, artifact_to_art)))
        sudo(upload)

        print ("Chown {} to JBoss's user...n".format(os.path.join(env.project_root, artifact_to_art)))
        sudo(chown)

        print ('Done:n')
        ch = run(getcontent)
        print ('{}n'.format(ch))


if __name__ == "__main__":

    getopts()

Вызов скрипта в Jenkins-е:

И пример деплоя результатов UI билда в Artifactory:

...
[workspace] $ /bin/sh -xe /tmp/hudson2349366895544039566.sh
+ python ci/deploy.py -a -t UI
Gradlefile: build.gradle
To Grab: version

VERSION: 1.0.0-SNAPSHOT

UI artifact found in f50500e-1.0.0-SNAPSHOT-unlock-ui.ear

Executing deploy f50500e-1.0.0-SNAPSHOT-unlock-ui.ear to https://project.artifactoryonline.com/project/project-releases-local/repo/ui/f50500e-1.0.0-SNAPSHOT-unlock-ui.ear.

Working now in /var/lib/jenkins/jobs/Project.UI.Build/workspace/unlock-ear/build/libs.

Artifacts ready to deployment:

f50500e-1.0.0-SNAPSHOT-unlock-ui.ear

Result code: 201

{
  "repo" : "project-releases-local",
  "path" : "/bub/ui/f50500e-1.0.0-SNAPSHOT-unlock-ui.ear",
  "created" : "2016-01-22T15:04:23.536Z",
  "createdBy" : "username",
  "downloadUri" : "https://project.artifactoryonline.com/project/project-releases-local/repo/ui/f50500e-1.0.0-SNAPSHOT-unlock-ui.ear",
  "mimeType" : "application/java-archive",
  "size" : "11318654",
  "checksums" : {
    "sha1" : "6f0f567f609578974245d7226bc87784d7b35edd",
    "md5" : "a4c899f08c1017a81fbaa9c62ebbd6e5"
  },
  "originalChecksums" : {
  },
  "uri" : "https://project.artifactoryonline.com/project/project-releases-local/repo/ui/f50500e-1.0.0-SNAPSHOT-unlock-ui.ear"
}

Triggering projects: Project.UI.Deploy.VMD
Finished: SUCCESS