import os import json from collections import OrderedDict from actions_logging.app_logging import logger from env_files.utils import get_sub_dir_for_env_file from env_files.validate_env_vars import check_if_file_exists from github.env import exit_on_error_and_write_summary, get_required_env_var, write_github_summary from aws.constants import PROD_ENVS, PRODUCTION, STG_PROD_ENVS from env_files.constants import DEFAULT_ENV_FILES_PATH, GLOBAL_FILE from env_files.json_template_handling import replace_dollar_values from core.handle_vault_env_var_for_sx_dns import merge_vault_env_var from common.common import run_command from github.get_file_from_github import get_file_from_github from git.utils import get_commit_hash from master.calculate_version import get_version def get_secrets_and_vars_from_json(json_data: dict) -> tuple[dict, dict]: try: secrets = {} vars = {} for key,val in json_data.items(): if isinstance(val, str) and val.startswith('SECRET_'): secret_name = val[7:] secrets[key] = secret_name else: vars[key] = val return secrets, vars except Exception as e: raise RuntimeError(f"error in get_secrets_and_vars_from_json: {e}") def handle_secrets(env_name, service_name): # save all the pulled secrets to a dict try: env = os.environ secrets_dict = {} env_level = env_name if env_name in PROD_ENVS: env_level = PRODUCTION upper_env_name = env_level.upper().replace("-", "_") upper_service_name = service_name.upper().replace("-", "_") global_cicd_prefix = f"GLOBAL_CICD_" global_keys_prefix = f"GLOBAL_KEYS_" global_certificates_prefix = f"GLOBAL_CERTIFICATES_" global_svc_prefix = f"GLOBAL_{upper_service_name}_" env_svc_prefix = f"ENV_{upper_env_name}_{upper_service_name}_" env_prefix = f"ENV_{upper_env_name}_" sorted_env = dict(sorted(env.items(), key=lambda x: x[0], reverse=True)) for key in sorted_env: if key.startswith(global_cicd_prefix): real_key = key.replace(global_cicd_prefix, "") secrets_dict[real_key] = env[key] elif key.startswith(global_keys_prefix): real_key = key.replace(global_keys_prefix, "") secrets_dict[real_key] = env[key] elif key.startswith(global_certificates_prefix): real_key = key.replace(global_certificates_prefix, "") secrets_dict[real_key] = env[key] elif key.startswith(global_svc_prefix): real_key = key.replace(global_svc_prefix, "") secrets_dict[real_key] = env[key] elif key.startswith(env_svc_prefix): real_key = key.replace(env_svc_prefix, "") secrets_dict[real_key] = env[key] elif key.startswith(env_prefix): if key.startswith(env_svc_prefix): real_key = key.replace(env_svc_prefix, "") secrets_dict[real_key] = env[key] else: real_key = key.replace(env_prefix, "") secrets_dict[real_key] = env[key] return secrets_dict except Exception as e: raise RuntimeError(f"error in handle_secrets: {e}") def iterate_secrets_and_env_vars(secrets_dict_from_env, secret_names, env_name): try: secrets_dict = {} for secret_name in secret_names: secret_name_value = secret_names[secret_name] if secret_name_value in secrets_dict_from_env.keys(): secrets_dict[secret_name] = secrets_dict_from_env[secret_name_value] else: message = f"secret SECRET_{secret_name_value} not found in environment {env_name}" if env_name in STG_PROD_ENVS: exit_on_error_and_write_summary(message) logger.warning(f"{message}. will set it to empty") write_github_summary(f":warning: {message}") secrets_dict[secret_name] = "" return secrets_dict except Exception as e: raise RuntimeError(f"error in iterate_secrets_and_env_vars: {e}") def create_env_file(final_vars, file_name): """ will create a dot env file from a dict :param final_vars: :param file_name: :return: """ try: vars = OrderedDict(sorted(final_vars.items())) with open(file_name, "w") as base_file: for k,v in vars.items(): if v == None: # if not done key=None in resulting .env base_file.write(f"{k}=\n") base_file.write(f"{k}={v}\n") logger.info(f"file {file_name} created successfully") except Exception as e: raise RuntimeError(f"error in create_env_file: {e}") def parse_secrets_and_vars_into_final_vars(vars_dict, secret_names, env_name, svc_name_for_secrets): try: secrets_dict_from_env = handle_secrets(env_name, svc_name_for_secrets) secrets_dict = iterate_secrets_and_env_vars(secrets_dict_from_env, secret_names, env_name) logger.debug(f"secrets_dict: {secrets_dict}") final_vars = {**vars_dict, **secrets_dict} logger.debug(f"final vars merged with vars_dict: {final_vars}") svc_type = os.getenv('SVC_TYPE', '') logger.debug(f"svc_type: {svc_type}, svc_name_for_secrets: {svc_name_for_secrets}, final_vars: {final_vars}") if svc_type == 'core-ecs' and svc_name_for_secrets == 'sx-core-dns' and 'VAULT' in final_vars: final_vars['VAULT'] = merge_vault_env_var(final_vars['VAULT'], secrets_dict_from_env) return final_vars except Exception as e: raise RuntimeError(f"error in parse_secrets_and_vars_into_final_vars: {e}") def inject_dd_env_vars_to_dot_env(svc_name: str, version: str) -> dict: """ Injects the Datadog environment variables into the vars_dict :param vars_dict: The dictionary of environment variables :param version: The version of the service :return: The dictionary of environment variables with the Datadog environment variables injected """ try: PACKAGE_JSON = 'package.json' repo_name = get_required_env_var('GITHUB_REPOSITORY') version = os.getenv("VERSION_TO_DEPLOY") env_name = get_required_env_var("ENV_NAME") commit_sha = get_commit_hash(short=False) repo_url = f"github.com/{repo_name}" runtime = os.getenv("RUNTIME") logger.info(f"detected RUNTIME: {runtime!r}") # only download package.json and reset commit_sha for non-Java runtimes if runtime != "Java" and get_version(version): logger.info(f"downloading {PACKAGE_JSON} from tag {version} since the version is a semver or hotfix") commit_sha_ref = run_command(f"git show-ref --tags refs/tags/{version}").strip() commit_sha = commit_sha_ref.split()[0] get_file_from_github(repo_name, PACKAGE_JSON, os.getcwd(), version) if not version: build_num = get_required_env_var("GITHUB_RUN_NUMBER") commit_sha_short = get_commit_hash() version = "-".join([env_name, build_num, commit_sha_short]) saferx_db_api_version='unknown' saferx_lib_version='unknown' # only extract saferx versions for non-Java runtimes if runtime != "Java": saferx_db_api_version = run_command( f"cat {PACKAGE_JSON} | jq -r " f"'.dependencies.\"saferx-dbapi\"' " f"| cut -d \"\\\"\" -f 4 | cut -d \\# -f 2" ).strip() logger.debug(f"saferx_db_api_version: {saferx_db_api_version}") saferx_lib_version = run_command( f"cat {PACKAGE_JSON} | jq -r '.dependencies.saferxlib' " f"| cut -d \"\\\"\" -f 4 | cut -d \\# -f 2" ).strip() logger.debug(f"saferx_lib_version: {saferx_lib_version}") # delete existing DD_VERSION and DD_TAGS from .env file before injecting new values, # awk is platform agnostic (sed is not) run_command(f"awk '!/^DD_VERSION=/ && !/^DD_TAGS=/' {svc_name}.env > temp && mv temp {svc_name}.env") with open(f'{svc_name}.env', 'a') as env_file: env_file.write(f"DD_VERSION={version}\n") # build and write DD_TAGS: always include git info; only include saferx tags if not Java tags = [] if runtime != "Java": tags.append(f"saferxLib:{saferx_lib_version}") tags.append(f"saferxDbapi:{saferx_db_api_version}") tags.extend([ f"git.commit.sha:{commit_sha}", f"git.repository_url:{repo_url}" ]) env_file.write(f"DD_TAGS={','.join(tags)}\n") logger.info("injected dd vars into .env file") except Exception as e: raise RuntimeError(f"couldn't inject dd variables to .env file: {e}") def merge_json_files(base_file_path: str, overrides_json_path: str) -> dict: """ Merges two JSON files into one dictionary. :param overrides_json_path: :param base_file_path: :return: dictionary of merged JSON files """ try: check_if_file_exists(overrides_json_path) check_if_file_exists(base_file_path) with open(overrides_json_path, 'r') as overrides_file: overrides = json.load(overrides_file) with open(base_file_path, 'r') as base_file: base = json.load(base_file) merged = {**base, **overrides} return merged except Exception as e: raise RuntimeError(f"error in merge_json_files: {e}") def create_dotenv_data(env_json_path: str, env_name: str, svc_name: str, is_multi_service_repo: bool, base_env_files_path: str = DEFAULT_ENV_FILES_PATH) -> dict: """ generate the json data of the .env file. before it will be .env this function will merge the env json file with the global json file and then rendering the secrets from environment variables and insert it to the json data before it becomes .env :param env_json_path: :param env_name: :param svc_name: :param is_multi_service_repo: :param base_env_files_path: :return: dict of the merged json files """ try: sub_dir = get_sub_dir_for_env_file(env_name) env_files_location = base_env_files_path if is_multi_service_repo: env_files_location = os.path.join(base_env_files_path, svc_name) global_file_path = os.path.join(env_files_location, GLOBAL_FILE) if not env_json_path: env_json_path = os.path.join(env_files_location, sub_dir, f"{env_name}.json") logger.debug(f"global file path: {global_file_path}") logger.debug(f"env file path: {env_json_path}") merged_json_data = merge_json_files(global_file_path, env_json_path) secret_names, vars_dict = get_secrets_and_vars_from_json(merged_json_data) logger.debug(f"done getting secrets and vars from json files {env_json_path} and {global_file_path}") # TODO: we still need this? svc_name_for_secrets = os.getenv('SVC_NAME', svc_name) final_vars = parse_secrets_and_vars_into_final_vars(vars_dict, secret_names, env_name, svc_name_for_secrets) final_vars = replace_dollar_values(final_vars) logger.info(f"done preparing final vars from jsons {env_json_path} and {global_file_path}") return final_vars except Exception as e: raise RuntimeError(f"error in create_dot_env_from_json: {e}") def create_dot_env_from_json(source_json_path: str, env_name: str, svc_name: str, is_multi_service_repo: bool, base_env_files_path: str = DEFAULT_ENV_FILES_PATH) -> str: """ takes env json file and global json file and creates a .env file :param source_json_path: :param env_name: :param svc_name: :param is_multi_service_repo: :param base_env_files_path: :return: filename of the created .env file """ try: logger.info(f"creating .env file for {env_name} {svc_name}") dotenv_data = create_dotenv_data(source_json_path, env_name, svc_name, is_multi_service_repo, base_env_files_path=base_env_files_path) file_name = f"{svc_name}.env" logger.debug(f"creating .env file {file_name}") create_env_file(dotenv_data, file_name) return file_name except Exception as e: raise RuntimeError(f"error in create_dot_env_from_json: {e}")