Commit 2017b026 authored by Tulio Ruiz's avatar Tulio Ruiz
Browse files

[REF] Add separete command for test and build image

[FIX] Better way to pass the parameters

[FIX] Add random string to filter better the containers

[FIX] Container name and add more information

[FIX] Give some time to allow postgres container to start

[FIX] Retry when the postgres container throw some error at the start
parent b17d4324
Pipeline #54372 passed with stage
in 26 seconds
......@@ -3,11 +3,11 @@
import click
import logging
from deployv_addon_gitlab_tools.commands import (
check_keys, test_images, upload_image, deployv_tests, push_coverage
check_keys, test_images, upload_image, deployv_tests, push_coverage, build_image, test_image
)
__version__ = "0.2.23"
logger = logging.getLogger('deployv.' + __name__) # pylint: disable=C0103
_logger = logging.getLogger('deployv.' + __name__)
@click.group(invoke_without_command=True)
......@@ -17,7 +17,7 @@ def gitlab_tools(ctx):
"""
ctx.command.config = ctx.parent.command.config
parent = ctx.parent.params
logger.log(
_logger.log(
getattr(logging, parent.get('log_level', 'INFO')),
"Deployv addon: Gitlab tools (ver: %s).",
__version__
......@@ -25,13 +25,17 @@ def gitlab_tools(ctx):
if not ctx.invoked_subcommand:
click.echo(ctx.get_help())
for command in [
check_keys.check_keys,
test_images.test_images,
upload_image.upload_image,
deployv_tests.deployv_tests,
push_coverage.push_coverage,
]:
build_image.build_image,
test_image.test_image
]:
gitlab_tools.add_command(command)
......
# coding: utf-8
import click
from deployv_addon_gitlab_tools import common
import logging
from os import environ
import signal
import sys
_logger = logging.getLogger('deployv.' + __name__)
signal.signal(signal.SIGTERM, common.reciveSignal)
signal.signal(signal.SIGINT, common.reciveSignal)
@click.command()
@click.option('--ci_commit_ref_name', default=environ.get('CI_COMMIT_REF_NAME'),
help=("The branch or tag name for which project is built."
" Env var: CI_COMMIT_REF_NAME."))
@click.option('--ci_pipeline_id', default=environ.get('CI_PIPELINE_ID'),
help=("The unique id of the current pipeline that GitLab CI"
" uses internally. Env var: CI_PIPELINE_ID."))
@click.option('--ci_repository_url', default=environ.get('CI_REPOSITORY_URL'),
help=("The URL to clone the Git repository."
" Env var: CI_REPOSITORY_URL."))
@click.option('--base_image', default=environ.get('BASE_IMAGE'),
help=("Env var: BASE_IMAGE."))
@click.option('--odoo_repo', default=environ.get('ODOO_REPO'),
help=("Env var: ODOO_REPO."))
@click.option('--odoo_branch', default=environ.get('ODOO_BRANCH'),
help=("Env var: ODOO_BRANCH."))
@click.option('--version', default=environ.get('VERSION'),
help=("Env var: VERSION."))
@click.option('--install', default=environ.get('MAIN_APP'),
help=("Env var: MAIN_APP."))
@click.option('--ci_job_id', default=environ.get('CI_JOB_ID'),
help=("The unique id of the current job that GitLab CI uses internally."
" Env var: CI_JOB_ID."))
@click.option('--psql_image', default=False,
help=("Override the default postgresql image to use for the tests"
"(Notice that this will override the PSQL_VERSION too)"))
@click.option('--image_repo_url', default=environ.get('IMAGE_REPO_URL', "quay.io/vauxoo"),
help=("The URL where the image repository is located."
" Env var: IMAGE_REPO_URL."))
@click.option('--push_image', is_flag=True,
help="If set it will push the image when on the main branch after the tests")
def build_image(**kwargs):
config = common.prepare(**kwargs)
if config.get('push_image'):
if not config.get('orchest_registry', False) or not config.get('orchest_token', False):
_logger.error('To push the image you need to set ORCHEST_REGISTRY and ORCHEST_TOKEN env vars')
sys.exit(1)
common.clean_containers(config)
common.pull_images([config['base_image'], ])
common.run_build_image(config)
is_latest = False
if config.get('push_image'):
# TODO: if we decide to build and push every image, just move the _IMAGE_TAG outside the if
if config.get('ci_commit_ref_name') == config['version']:
common.push_image(config, config['instance_image'], 'latest')
is_latest = True
common.push_image(config, config['instance_image'], config['image_tag'])
common.notify_orchest(config, is_latest=is_latest)
common.save_imagename(config)
common.clear_images(config)
sys.exit(0)
# coding: utf-8
import click
import distutils.spawn
from deployv.helpers import utils
import shlex
import subprocess
from deployv_addon_gitlab_tools.common import check_credentials, prepare
import logging
from os import path, makedirs, environ, chmod
from os import environ
logger = logging.getLogger('deployv.' + __name__) # pylint: disable=C0103
TO_SCAN = ['git.vauxoo.com',
'github.com']
def check_ssh_folder():
""" Check if the folder exists and create it
:return: The full path to the .ssh folder
"""
home_path = path.expanduser(path.join('~', '.ssh'))
if not path.isdir(home_path):
makedirs(home_path)
return home_path
def add_private_key(key, folder):
""" Generates the id_rsa file if it doesn't exits with the proper
format and permissions
:param key: The key content
:param folder: The folder where the id_rsa file will be stored
"""
ssh_file = path.join(folder, 'id_rsa')
if path.isfile(ssh_file):
logger.info('The id_rsa file already exists, nothing to do')
return
with open(ssh_file, 'w') as ssh_key:
ssh_key.write(key)
try:
subprocess.check_call(['dos2unix', ssh_file])
except subprocess.CalledProcessError:
logger.error('You need to install dos2unix to check the key')
chmod(ssh_file, 0o0600)
def scan_keys(folder):
""" Performs a ssk-key scan in the list of hosts and add the keys to the
known_hosts files
:param folder: The folder where the file will be stored
"""
known_hosts = path.join(folder, 'known_hosts')
with open(known_hosts, 'a') as known_file:
for host in TO_SCAN:
keys = subprocess.check_output(['ssh-keyscan', host], stderr=subprocess.STDOUT)
for line in utils.decode(keys).split('\n'):
clean = line.strip()
if clean:
known_file.write(clean + '\n')
subprocess.check_call(['ls', '-l', folder])
def check_docker():
""" Checks if the docker binary is present in the running environment """
return distutils.spawn.find_executable("docker")
def is_docker_login():
""" Check if we have all we need to docker login (via cli or api), that is:
- We have docker binary available
- We have the env vars properly set (DOCKER_PASSWORD and DOCKER_USER)"""
return check_docker() and environ.get('DOCKER_PASSWORD', False) and environ.get('DOCKER_USER', False)
def docker_login():
""" Execute docker login from the actual console because whe done via the api they won't persist """
cmd = 'sh -c "echo ${DOCKER_PASSWORD} | docker login --username ${DOCKER_USER} --password-stdin quay.io"'
subprocess.Popen(shlex.split(cmd))
def check_credentials(private_deploy_key):
ssh_folder = check_ssh_folder()
if private_deploy_key:
add_private_key(private_deploy_key, ssh_folder)
scan_keys(ssh_folder)
if is_docker_login():
docker_login()
_logger = logging.getLogger('deployv.' + __name__)
@click.command()
......@@ -98,4 +16,6 @@ def check_credentials(private_deploy_key):
def check_keys(private_deploy_key):
"""Checks if the .ssh folder exists, creates it and add the private key
if necessary"""
check_credentials(private_deploy_key)
_logger.info('Check keys command')
config = prepare(private_deploy_key=private_deploy_key)
check_credentials(config)
# coding: utf-8
import click
from deployv_addon_gitlab_tools import common
from os import environ
import sys
import logging
import signal
_logger = logging.getLogger('deployv.' + __name__)
signal.signal(signal.SIGTERM, common.reciveSignal)
signal.signal(signal.SIGINT, common.reciveSignal)
@click.command()
@click.option('--ci_commit_ref_name', default=environ.get('CI_COMMIT_REF_NAME'),
help=("The branch or tag name for which project is built."
" Env var: CI_COMMIT_REF_NAME."))
@click.option('--ci_pipeline_id', default=environ.get('CI_PIPELINE_ID'),
help=("The unique id of the current pipeline that GitLab CI"
" uses internally. Env var: CI_PIPELINE_ID."))
@click.option('--ci_repository_url', default=environ.get('CI_REPOSITORY_URL'),
help=("The URL to clone the Git repository."
" Env var: CI_REPOSITORY_URL."))
@click.option('--base_image', default=environ.get('BASE_IMAGE'),
help=("Env var: BASE_IMAGE."))
@click.option('--odoo_repo', default=environ.get('ODOO_REPO'),
help=("Env var: ODOO_REPO."))
@click.option('--odoo_branch', default=environ.get('ODOO_BRANCH'),
help=("Env var: ODOO_BRANCH."))
@click.option('--version', default=environ.get('VERSION'),
help=("Env var: VERSION."))
@click.option('--install', default=environ.get('MAIN_APP'),
help=("Env var: MAIN_APP."))
@click.option('--ci_job_id', default=environ.get('CI_JOB_ID'),
help=("The unique id of the current job that GitLab CI uses internally."
" Env var: CI_JOB_ID."))
@click.option('--psql_image', default=False,
help=("Override the default postgresql image to use for the tests"
"(Notice that this will override the PSQL_VERSION too)"))
@click.option('--image_repo_url', default=environ.get('IMAGE_REPO_URL', "quay.io/vauxoo"),
help=("The URL where the image repository is located."
" Env var: IMAGE_REPO_URL."))
def test_image(**kwargs):
config = common.prepare(**kwargs)
common.pull_images([config['instance_image'],
config['postgres_image']])
res = common.run_image_tests(config)
if not res:
common.clear_images(config)
sys.exit(1)
common.clear_images(config)
sys.exit(0)
# coding: utf-8
from deployv_addon_gitlab_tools.common import check_env_vars, get_main_app
from deployv_addon_gitlab_tools.commands.check_keys import check_credentials, is_docker_login
from deployv.helpers import utils
from docker import errors, APIClient as Client
import click
from deployv_addon_gitlab_tools import common
from os import environ
import re
import subprocess
import time
import shlex
import sys
import click
import logging
import json
import requests
import signal
from urllib3.exceptions import ReadTimeoutError
logger = logging.getLogger('deployv.' + __name__) # pylint: disable=C0103
_cli = Client(timeout=7200)
def reciveSignal(signalNumber, frame):
clean_containers()
clear_images()
sys.exit(0)
signal.signal(signal.SIGTERM, reciveSignal)
signal.signal(signal.SIGINT, reciveSignal)
def generate_image_name(name):
""" Generate the base image name usig the ref name but cleaning it before,
ATM only removes "." and "#" from the title to avoid issues with docker naming
convention """
res = re.sub(r'[\.#\$\=\+\;\>\,\<,\&\%]', '', name)
res = re.sub(r'-_', '_', res)
return res.lower()
def build_image():
logger.info('Building image')
cmd = ('deployvcmd build -b {branch} -u {url} -v {version} -i {image} -O {repo}#{odoo_branch} -T {tag}'
.format(branch=environ['CI_COMMIT_REF_NAME'], url=environ['CI_REPOSITORY_URL'], repo=environ['ODOO_REPO'],
odoo_branch=environ['ODOO_BRANCH'], version=environ['VERSION'], image=environ['BASE_IMAGE'],
tag=environ['_INSTANCE_IMAGE']))
try:
subprocess.check_call(shlex.split(cmd))
except subprocess.CalledProcessError:
logger.exception('Could not build the image, please read the log above')
sys.exit(1)
images = _cli.images(environ['_INSTANCE_IMAGE'])
image_sha = images[0].get('Id')
short_id = image_sha.split(':')[1][:10]
environ.update({
'_IMAGE_TAG': short_id,
})
def pull_images():
""" Pulls images needed for the build and test process """
images = [environ['BASE_IMAGE'],
environ['_POSTGRES_IMAGE']]
for image in images:
logger.info('Pulling: %s', image)
_cli.pull(image)
return images
def postgres_container():
return 'postgres{0}_{1}'.format(environ['_BASE_NAME'], environ['CI_PIPELINE_ID'])
def clean_containers():
""" Cleans any running container related to the same build to avoid any conflicts """
containers = _cli.containers(all=True, filters={'name': environ['_BASE_NAME']})
for container in containers:
try:
logger.info('Removing container %s', container.get('Name', container.get('Names')[0]))
_cli.remove_container(container['Id'], force=True)
except errors.NotFound:
logger.info('Container %s does not exist', container.get('Name', container.get('Names')[0]))
def clear_images():
logger.info('Removing image %s', environ['_INSTANCE_IMAGE'])
try:
_cli.remove_image(environ['_INSTANCE_IMAGE'])
except errors.APIError as error:
if 'No such image' in error.explanation:
pass
logger.info('Image %s deleted', environ['_INSTANCE_IMAGE'])
def start_postgres():
logger.info('Starting container %s', postgres_container())
container = _cli.create_container(image=environ['_POSTGRES_IMAGE'],
name=postgres_container(),
environment={'POSTGRES_PASSWORD': 'postgres'})
_cli.start(container=container.get('Id'))
logger.info(container)
def create_postgres_user():
cmd = "psql -c \"create user odoo with password 'odoo' createdb\""
res = exec_cmd(postgres_container(), cmd, 'postgres')
return res
def start_instance():
env = {
"DB_USER": "odoo",
"DB_PASSWORD": "odoo",
"DB_HOST": postgres_container(),
"ODOO_CONFIG_FILE": "/home/odoo/.openerp_serverrc"
}
for env_var in ['COUNTRY', 'LANGUAGE']:
env.update({env_var: environ.get(env_var, "")})
links = {
postgres_container(): postgres_container()
}
host_config = _cli.create_host_config(links=links)
logger.info('Starting container %s', environ['_INSTANCE_IMAGE'])
logger.debug('Env vars %s', json.dumps(env, sort_keys=True, indent=4))
container = _cli.create_container(image=environ['_INSTANCE_IMAGE'],
name=environ['_INSTANCE_IMAGE'],
environment=env,
host_config=host_config)
_cli.start(container=container.get('Id'))
logger.info(container)
_logger = logging.getLogger('deployv.' + __name__)
def install_module():
module = get_main_app()
extra = ''
if environ.get('LANGUAGE'):
extra += ' --load-language={lang}'.format(lang=environ.get('LANGUAGE'))
install_wdemo = (
"/home/odoo/instance/odoo/odoo-bin -d wdemo -i {mod}"
"{extra} --stop-after-init".format(mod=module, extra=extra)
)
install_wodemo = (
"/home/odoo/instance/odoo/odoo-bin -d wodemo -i {mod}"
"{extra} --stop-after-init --without-demo=all".format(mod=module, extra=extra)
)
logger.info('Verifying supervisorctl')
is_running()
logger.info('Stopping odoo')
exec_cmd(environ['_INSTANCE_IMAGE'], 'supervisorctl stop odoo')
logger.info('\nInstalling %s with demo', module)
logger.debug('Command : %s', install_wdemo)
wdemo_res = exec_cmd(environ['_INSTANCE_IMAGE'], install_wdemo, 'odoo', stream=True)
wdemo_log = resume_log(wdemo_res)
logger.info('\nInstalling %s without demo', module)
logger.debug('Command : %s', install_wodemo)
wodemo_res = exec_cmd(environ['_INSTANCE_IMAGE'], install_wodemo, 'odoo', stream=True)
wodemo_log = resume_log(wodemo_res)
show_log(wdemo_log[1], 'Installation with demo')
show_log(wodemo_log[1], 'Installation without demo')
if not wdemo_log[0] or not wodemo_log[0]:
return False
return True
def exec_cmd(container, cmd, user=None, stream=False):
lines = []
container_id = _cli.inspect_container(container).get('Id')
logger.debug('Executing command "{cmd}" in container "{con}".'.format(cmd=cmd, con=container))
try:
exec_id = _cli.exec_create(container_id, cmd, user=user)
except errors.APIError as error:
logger.error('Error: %s', error.explanation)
raise
res = _cli.exec_start(exec_id.get('Id'), stream=stream)
if stream:
for line in res:
line = utils.decode(line)
logger.info(line.strip('\n'))
lines.append(line)
return lines
return utils.decode(res)
def show_log(log, title):
logger.info('\n%s', title)
logger.info('='*20)
logger.info('+-- Critical errors %s', len(log.get('critical')))
logger.info('+-- Errors %s', len(log.get('errors')))
logger.info('+-- Import errors %s', len(log.get('import_errors')))
logger.info('+-- Warnings %s', len(log.get('warnings')))
logger.info('+-- Translation Warnings %s', len(log.get('warnings_trans')))
logger.info('='*20)
def resume_log(log_lines):
"""Gets the log lines from -u (modules or all) and parse them to get the totals
according to the filters dict
:param log_lines: each element of the list is a log line
:return: dict with key filters as keys and a list with all matched lines
"""
def critical(line):
criteria = re.compile(r'.*\d\sCRITICAL\s.*')
return criteria.match(line)
def errors(line):
criteria = re.compile(r'.*\d\sERROR\s.*')
return criteria.match(line)
def warnings_trans(line):
criteria = re.compile(
r'.*\d\sWARNING\s.*no translation for language.*')
return criteria.match(line)
def import_errors(line):
criteria = re.compile(r'^ImportError.*')
return criteria.match(line)
def warnings(line):
criteria = re.compile(r'.*\d\sWARNING\s.*')
return criteria.match(line) and 'no translation for language' not in line
filters = {
'critical': critical,
'errors': errors,
'warnings': warnings,
'warnings_trans': warnings_trans,
'import_errors': import_errors
}
success = True
res = {name: [] for name in filters}
for line in log_lines:
stripped_line = line.strip()
for name, criteria in filters.items():
if criteria(stripped_line):
if name in ['critical', 'errors']:
success = False
elif name == 'warnings' and 'Deprecated' in stripped_line:
success = False
res.get(name).append(stripped_line)
break
return (success, res)
def is_running():
retry = True
retries = 0
while retry and retries <= 10:
try:
res = exec_cmd(environ['_INSTANCE_IMAGE'], 'supervisorctl status odoo')
except errors.APIError:
retries += 1
logger.warn('Container error, retrying %s', retries)
time.sleep(5)
continue
logger.info('is_running: %s', res.strip())
if 'STARTING' in res or 'STOPPING' in res:
logger.warn('The Odoo process is in an intermediate state, retrying')
time.sleep(5)
elif 'RUNNING' in res:
return True
elif 'STOPPED' in res:
return False
elif res == '' or 'no such file' in res:
retries += 1
logger.warn('Supervisor returned empty or not running yet, retrying %s', retries)
time.sleep(5)
else:
retries += 1
logger.warn('Unknown state: %s', res)
time.sleep(5)
def push_image(image_name, image_tag):
logger.info('Pushing image %s to %s:%s', image_name, environ['_IMAGE_REPO'], image_tag)
_cli.tag(image_name, environ['_IMAGE_REPO'], tag=image_tag)
if is_docker_login():
_cli.login(environ['DOCKER_USER'], environ['DOCKER_PASSWORD'], registry='quay.io')
for attempt in range(4):
try:
for result in _cli.push(environ['_IMAGE_REPO'], tag=image_tag, stream=True):
result = json.loads(utils.decode(result))
if result.get('error'):
logger.error(result.get('error'))
sys.exit(1)
else:
break
except ReadTimeoutError as error:
if 'Read timed out' in error.message and attempt < 3:
logger.warn('An error raised while pushing the image, retrying (%s / 3)', attempt+1)
else:
raise
logger.info('Image pushed correctly')
def notify_orchest(tag, customer, is_latest=False):
image_name = '{image}:{tag}'.format(image=environ['_IMAGE_REPO'], tag=tag)
res = requests.post(
environ['ORCHEST_REGISTRY'], data=json.dumps({