seecoder-paas/src/main/java/cn/seecoder/paas/service/impl/EnvironmentServiceImpl.java

package cn.seecoder.paas.service.impl;

import cn.seecoder.paas.data.dao.ApplicationDAO;
import cn.seecoder.paas.data.dao.ConfigDAO;
import cn.seecoder.paas.data.dao.EnvironmentDAO;
import cn.seecoder.paas.data.entity.Application;
import cn.seecoder.paas.data.entity.Config;
import cn.seecoder.paas.data.entity.Environment;
import cn.seecoder.paas.service.EnvironmentService;
import cn.seecoder.paas.service.facade.docker.DockerApi;
import cn.seecoder.paas.service.facade.git.GitApi;
import cn.seecoder.paas.service.facade.k8s.*;
import cn.seecoder.paas.service.facade.k8s.model.*;
import cn.seecoder.paas.service.facade.k8s.vo.SecretVO;
import cn.seecoder.paas.service.model.converter.ConfigConverter;
import cn.seecoder.paas.service.model.converter.EnvironmentConverter;
import cn.seecoder.paas.service.model.vo.ConfigVO;
import cn.seecoder.paas.service.model.vo.EnvironmentVO;
import cn.seecoder.paas.util.ApplicationProperties;
import cn.seecoder.paas.util.AsyncWrapper;
import cn.seecoder.paas.util.LoggerUtil;
import cn.seecoder.paas.util.ServiceException;
import cn.seecoder.paas.util.enums.*;
import com.spotify.docker.client.ProgressHandler;
import com.spotify.docker.client.exceptions.DockerException;
import com.spotify.docker.client.messages.ProgressMessage;
import io.kubernetes.client.openapi.models.*;
import org.apache.commons.lang.StringUtils;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.ResetCommand;
import org.slf4j.Logger;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.transaction.support.TransactionSynchronizationAdapter;
import org.springframework.transaction.support.TransactionSynchronizationManager;
import org.springframework.util.CollectionUtils;

import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.util.*;
import java.util.stream.Collectors;

@org.springframework.stereotype.Service
public class EnvironmentServiceImpl implements EnvironmentService {

    private static final Logger logger = LoggerUtil.getLogger(EnvironmentServiceImpl.class);

    private final EnvironmentDAO environmentDAO;

    private final ApplicationDAO applicationDAO;

    private final ConfigDAO configDAO;

    private final PodApi podApi;

    private final DeploymentApi deploymentApi;

    private final ServiceApi serviceApi;

    private final IngressApi ingressApi;

    private final SecretApi secretApi;

    private final ConfigMapApi configMapApi;

    private final AsyncWrapper asyncWrapper;

    private final GitApi gitApi;

    private final DockerApi dockerApi;

    private final LogApi logApi;

    private final ApplicationProperties applicationProperties;

    private static final String DEFAULT_IMAGE_PULL_SECRET_NAME = "seecoder-paas-image-pull-secret";

    @Autowired
    public EnvironmentServiceImpl(EnvironmentDAO environmentDAO,
                                  ApplicationDAO applicationDAO,
                                  ConfigDAO configDAO,
                                  PodApi podApi,
                                  DeploymentApi deploymentApi,
                                  ServiceApi serviceApi,
                                  IngressApi ingressApi,
                                  SecretApi secretApi,
                                  ConfigMapApi configMapApi,
                                  GitApi gitApi,
                                  DockerApi dockerApi,
                                  LogApi logApi,
                                  ApplicationProperties properties,
                                  AsyncWrapper asyncWrapper) {
        this.environmentDAO = environmentDAO;
        this.applicationDAO = applicationDAO;
        this.configDAO = configDAO;
        this.podApi = podApi;
        this.deploymentApi = deploymentApi;
        this.serviceApi = serviceApi;
        this.ingressApi = ingressApi;
        this.secretApi = secretApi;
        this.configMapApi = configMapApi;
        this.gitApi = gitApi;
        this.dockerApi = dockerApi;
        this.logApi = logApi;
        this.applicationProperties = properties;
        this.asyncWrapper = asyncWrapper;
    }

    @Override
    @Transactional
    public EnvironmentVO createOrUpdate(EnvironmentVO environmentVO) throws ServiceException {
        Application application = applicationDAO.findById(environmentVO.getAppId()).orElse(null);
        if (application == null) {
            throw ServiceException.BAD_REQUEST;
        }
        Environment environment = EnvironmentConverter.convertToEntity(environmentVO);
        if (environmentVO.getId() != null) {
            environment = environmentDAO.findById(environmentVO.getId()).orElse(null);
            if (environment == null) {
                throw ServiceException.BAD_REQUEST;
            }
            BeanUtils.copyProperties(environmentVO, environment, "id", "appId", "buildStatus", "buildOutput", "deployStatus", "image", "buildStartTime", "deployStartTime", "deployOutput");
        }
        EnvironmentVO result = EnvironmentConverter.convertToVO(environmentDAO.save(environment));
        ConfigVO resultConfigVO;
        if (environmentVO.getId() == null) {
            // 是create
            // 获取
            Config config = configDAO.findByConfigBelongsAndAndEntityId(ConfigBelongsToType.APPLICATION, application.getId());
            if (config == null) {
                config = ConfigConverter.convertToEntity(ConfigVO.getEmptyConfigVO(ConfigBelongsToType.ENVIRONMENT, result.getId()));
            }
            Config saved = new Config();
            BeanUtils.copyProperties(config, saved);
            saved.setId(null);
            saved.setConfigBelongs(ConfigBelongsToType.ENVIRONMENT);
            saved.setEntityId(result.getId());
            resultConfigVO = ConfigConverter.convertToVO(configDAO.save(saved));
        } else {
            resultConfigVO = ConfigConverter.convertToVO(configDAO.findByConfigBelongsAndAndEntityId(ConfigBelongsToType.ENVIRONMENT, result.getId()));
        }
        result.setConfig(resultConfigVO);
        String labelKey = ResourceLabel.RESOURCE.getCode();
        String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(result.getAppId()), String.valueOf(result.getId()));
        K8sObjectRequest request = K8sObjectRequest.builder().namespace(applicationProperties.getDeploymentNamespace()).labels(Collections.singletonMap(labelKey, labelValue)).build();
        List<Pod> pods = podApi.getByCondition(request);
        result.setInstance(pods);
        return result;
    }

    @Override
    @Transactional
    public void delete(Integer id) throws ServiceException {
        Environment environment = environmentDAO.findById(id).orElse(null);
        if (environment == null) {
            throw ServiceException.BAD_REQUEST;
        }
        environmentDAO.deleteById(id);
        configDAO.deleteByConfigBelongsAndAndEntityId(ConfigBelongsToType.ENVIRONMENT, id);
        String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(environment.getAppId()), String.valueOf(environment.getId()));
        String namespace = applicationProperties.getDeploymentNamespace();
        Map<String, String> labels = Collections.singletonMap(ResourceLabel.RESOURCE.getCode(), labelValue);
        AbstractApi[] deleteApis = {ingressApi, serviceApi, deploymentApi, configMapApi};
        for (AbstractApi api : deleteApis) {
            this.findAndDelete(api, namespace, labels);
        }
    }

    private void findAndDelete(AbstractApi abstractApi, String deployNamespace, Map<String, String> labels) {
        List<K8sAbstractObject> objects = abstractApi.getByCondition(K8sObjectRequest.builder().namespace(deployNamespace).labels(labels).build());
        if (!CollectionUtils.isEmpty(objects)) {
            for (K8sAbstractObject object : objects) {
                try {
                    abstractApi.delete(object);
                } catch (Exception e) {
                    LoggerUtil.error(logger, e, "deleteObjectError: object={}, labels={}", object, labels);
                }
            }
        }
    }

    @Override
    public EnvironmentVO get(Integer id, boolean fetchAll) throws ServiceException {
        Environment result = environmentDAO.findById(id).orElse(null);
        if (result == null) {
            throw ServiceException.BAD_REQUEST;
        }
        ConfigVO resultConfigVO = ConfigConverter.convertToVO(configDAO.findByConfigBelongsAndAndEntityId(ConfigBelongsToType.ENVIRONMENT, result.getId()));
        EnvironmentVO vo = EnvironmentConverter.convertToVO(result);
        if (fetchAll) {
            vo.setBuildOutput(result.getBuildOutput());
            vo.setDeployOutput(result.getDeployOutput());
        }
        vo.setConfig(resultConfigVO);
        String labelKey = ResourceLabel.RESOURCE.getCode();
        String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(result.getAppId()), String.valueOf(result.getId()));
        K8sObjectRequest request = K8sObjectRequest.builder().namespace(applicationProperties.getDeploymentNamespace()).labels(Collections.singletonMap(labelKey, labelValue)).build();
        List<Pod> pods = podApi.getByCondition(request);
        vo.setInstance(pods);
        return vo;
    }

    @Override
    public List<EnvironmentVO> getListByAppId(Integer appId) {
        return environmentDAO.findAllByAppId(appId).stream().map(EnvironmentConverter::convertToVO).collect(Collectors.toList());
    }

    @Override
    @Transactional
    public EnvironmentVO deploy(Integer id) throws ServiceException {
        Environment result = environmentDAO.findById(id).orElse(null);
        if (result == null) {
            throw ServiceException.BAD_REQUEST;
        }
        if (result.getBuildStatus() == BuildStatus.BUILDING || result.getDeployStatus() == DeployStatus.DEPLOYING || result.getDeployStatus() == DeployStatus.RESTARTING) {
            throw new ServiceException("101", "存在正在进行的构建或部署！");
        }
        Config config = configDAO.findByConfigBelongsAndAndEntityId(ConfigBelongsToType.ENVIRONMENT, id);
        if (config == null) {
            throw ServiceException.INVALID_DATA;
        }
        result.setBuildOutput("");
        result.setBuildStatus(BuildStatus.BUILDING);
        result.setBuildStartTime(LocalDateTime.now());
        result.setDeployStatus(DeployStatus.NOT_STARTED);
        TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronizationAdapter() {
            @Override
            public void afterCommit() {
                super.afterCommit();
                asyncWrapper.asyncInvoke(() -> buildAsync(id));
            }
        });
        return EnvironmentConverter.convertToVO(environmentDAO.save(result));
    }

    @Override
    public Map<String,String> getLog(Integer id, String podName) throws ServiceException {
        Environment result = environmentDAO.findById(id).orElse(null);
        if (result == null) {
            throw ServiceException.BAD_REQUEST;
        }
        String labelKey = ResourceLabel.RESOURCE.getCode();
        String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(result.getAppId()), String.valueOf(result.getId()));
        K8sObjectRequest request = K8sObjectRequest.builder().name(podName).namespace(applicationProperties.getDeploymentNamespace()).labels(Collections.singletonMap(labelKey, labelValue)).build();
        List<Pod> pods = podApi.getByCondition(request);
        if (CollectionUtils.isEmpty(pods)) {
            return Collections.emptyMap();
        }
        Map<String, String> res = new HashMap<>();
        for (Container container : pods.get(0).getContainers()) {
            res.put(container.getName(), logApi.getAll(applicationProperties.getDeploymentNamespace(), pods.get(0).getName(), container.getName()));
        }
        return res;
    }

    @Override
    @Transactional
    public EnvironmentVO restart(Integer id, String podName) throws ServiceException {
        Environment result = environmentDAO.findById(id).orElse(null);
        if (result == null) {
            throw ServiceException.BAD_REQUEST;
        }
        if (result.getBuildStatus() == BuildStatus.BUILDING || result.getDeployStatus() == DeployStatus.DEPLOYING || result.getDeployStatus() == DeployStatus.RESTARTING) {
            throw new ServiceException("101", "存在正在进行的构建或部署！");
        }
        Pod pod = new Pod();
        pod.setName(podName);
        pod.setNamespace(applicationProperties.getDeploymentNamespace());
        podApi.delete(pod);
        result.setDeployStatus(DeployStatus.RESTARTING);
        result.setDeployStartTime(LocalDateTime.now());
        return EnvironmentConverter.convertToVO(environmentDAO.save(result));
    }

    @Override
    @Transactional
    public void checkDeployStatus() {
        LocalDateTime startTime = LocalDateTime.now().minusMinutes(10);
        List<Environment> environments = environmentDAO.findAllByBuildStatusAndBuildStartTimeBefore(BuildStatus.BUILDING, startTime);
        if (!CollectionUtils.isEmpty(environments)) {
            for (Environment environment : environments) {
                environment.setBuildStatus(BuildStatus.FAIL);
                environment.setBuildOutput("Timeout interrupt.");
                logger.info("Interrupt buildTask: " + environment.getId());
                environmentDAO.save(environment);
            }
        }
        environments = environmentDAO.findAllByDeployStatus(DeployStatus.DEPLOYING);
        environments.addAll(environmentDAO.findAllByDeployStatus(DeployStatus.RESTARTING));
        if (!CollectionUtils.isEmpty(environments)) {
            for (Environment environment : environments) {
                if (environment.getDeployStartTime().isBefore(startTime)) {
                    environment.setDeployStatus(DeployStatus.FAIL);
                    environment.setDeployOutput("Timeout interrupt.");
                    logger.info("Interrupt deployTask: " + environment.getId());
                    environmentDAO.save(environment);
                } else {
                    String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(environment.getAppId()), String.valueOf(environment.getId()));
                    List<Deployment> deployments = deploymentApi.getByCondition(K8sObjectRequest
                            .builder()
                            .name(labelValue)
                            .namespace(applicationProperties.getDeploymentNamespace())
                            .build());
                    List<Pod> pods = podApi.getByCondition(K8sObjectRequest
                            .builder()
                            .labels(Collections.singletonMap(ResourceLabel.RESOURCE.getCode(), labelValue))
                            .namespace(applicationProperties.getDeploymentNamespace())
                            .build());
                    // 优先级先看pod，信息比较多
                    if (!CollectionUtils.isEmpty(pods)) {
                        V1PodStatus status = pods.get(0).getPodStatus();
                        List<V1PodCondition> conditions = status.getConditions() == null ? Collections.emptyList() : status.getConditions();
                        Collections.sort(conditions, Comparator.comparingLong(c -> ((V1PodCondition)c).getLastTransitionTime().getMillis()).reversed());
                        String deployOutput = "Type\tStatus\tMessage\tReason\tTime\n";
                        deployOutput += conditions.stream().map(condition -> {
                            return condition.getType() + "\t" + condition.getStatus() + "\t"  + condition.getMessage() + "\t" + condition.getReason() + "\t" + condition.getLastTransitionTime().toString();
                        }).collect(Collectors.joining("\n"));
                        switch (status.getPhase()) {
                            case "Succeeded":
                            case "Running":
                                if (conditions.get(0) != null && conditions.get(0).getStatus().equals("False")) {
                                    environment.setDeployStatus(DeployStatus.FAIL);
                                } else {
                                    environment.setDeployStatus(DeployStatus.SUCCESS);
                                }
                                break;
                            case "Pending":
                                environment.setDeployStatus(DeployStatus.DEPLOYING);
                                break;
                            case "Failed":
                            case "Unknown":
                            default:
                                environment.setDeployStatus(DeployStatus.FAIL);
                                break;
                        }
                        environment.setDeployOutput(deployOutput);
                        environmentDAO.save(environment);
                    }
                }
            }
        }
    }

    void buildAsync(Integer id) {
        Environment result = environmentDAO.findById(id).get();
        try {
            Config config = configDAO.findByConfigBelongsAndAndEntityId(ConfigBelongsToType.ENVIRONMENT, id);
            Application application = applicationDAO.findById(result.getAppId()).get();
            ConfigVO.ConfigContent configContent = ConfigConverter.convertToVO(config).getConfig();
            String labelKey = ResourceLabel.RESOURCE.getCode();
            String labelValue = ResourceLabel.RESOURCE.getGenerator().gen("environment", String.valueOf(result.getAppId()), String.valueOf(result.getId()));
            Map<String, String> labelsMap = Collections.singletonMap(labelKey, labelValue);
            String deploymentNamespace = applicationProperties.getDeploymentNamespace();
            String deploymentHost = applicationProperties.getDeploymentHost();
            String imageName = result.getBuildTypeValue();
            // 1. 有git从git拿到地址进行构建并push
            if (result.getBuildType() == BuildType.FROM_BRANCH_OR_COMMIT) {
                try {
                    Path directory = Files.createTempDirectory("seecoder-paas-");
                    Git git = gitApi.clone(application.getGitUrl(), directory.toString());
                    try {
                        // 尝试切换分支
                        git.checkout().setCreateBranch(false).setName(result.getBuildTypeValue()).call();
                    } catch (Exception gitException) {
                        // 否则尝试commit
                        git.reset().setMode(ResetCommand.ResetType.HARD).setRef(result.getBuildTypeValue()).call();
                    }
                    if (Files.exists(Paths.get(directory.toString(), "Dockerfile"))) {
                        String imageTag = String.valueOf(new Date().getTime());
                        StringBuilder sb = new StringBuilder();
                        final int DEFAULT_BUFFER_LENGTH = 200;
                        final long MAX_BUFFER_SIZE = 1 << 18;
                        dockerApi.buildAndPush(directory.toString(), "seecoder-paas-" + labelValue, imageTag, new ProgressHandler() {
                            @Override
                            public void progress(ProgressMessage message) throws DockerException {
                                String value = message.stream();
                                if (value == null) {
                                    value = "";
                                }
                                if (!StringUtils.isEmpty(message.error())) {
                                    value += "\n" + message.error();
                                }
                                sb.append(value);
                                if (sb.length() > DEFAULT_BUFFER_LENGTH) {
                                    result.setBuildOutput(result.getBuildOutput() + sb.toString());
                                    if (result.getBuildOutput().length() > MAX_BUFFER_SIZE) {
                                        result.setBuildOutput(result.getBuildOutput().substring((int) (result.getBuildOutput().length() - MAX_BUFFER_SIZE)));
                                    }
                                    sb.setLength(0);
                                    environmentDAO.save(result);
                                }
                            }
                        });
                        if (sb.length() > 0) {
                            result.setBuildOutput(result.getBuildOutput() + sb.toString());
                        }
                        imageName = applicationProperties.getK8s().getImageRegistry() + "/seecoder-paas-" + labelValue + ":" + imageTag;
                        result.setBuildStatus(BuildStatus.SUCCESS);
                        result.setDeployStatus(DeployStatus.DEPLOYING);
                        result.setDeployStartTime(LocalDateTime.now());
                        result.setImage(imageName);
                        environmentDAO.save(result);
                    } else {
                        result.setBuildStatus(BuildStatus.FAIL);
                        result.setBuildOutput("找不到构建文件Dockerfile!");
                        environmentDAO.save(result);
                        throw new ServiceException("102", "找不到构建文件Dockerfile!");
                    }
                } catch (Exception e) {
                    result.setBuildStatus(BuildStatus.FAIL);
                    if (!e.getLocalizedMessage().contains("Could not acquire image ID or digest following build")) {
                        result.setBuildOutput(e.getLocalizedMessage());
                    }
                    environmentDAO.save(result);
                    LoggerUtil.error(logger, e, "构建失败！gitUrl={}, branchOrCommit={}", application.getGitUrl(), result.getBuildTypeValue());
                    return;
                }
            } else if (result.getBuildType() == BuildType.FROM_IMAGE) {
                result.setBuildStatus(BuildStatus.SUCCESS);
                result.setBuildOutput("镜像构建无需部署");
                result.setDeployStatus(DeployStatus.DEPLOYING);
                result.setDeployStartTime(LocalDateTime.now());
                result.setImage(imageName);
                environmentDAO.save(result);
            }

            String[] ports = (configContent.getServicePort() == null ? "" : configContent.getServicePort()).split(",");
            if (!StringUtils.isEmpty(configContent.getHostPrefix())) {
                for (String port : ports) {
                    // ingress
                    List<Ingress> ingresses = ingressApi.getByCondition(K8sObjectRequest.builder()
                            .name(labelValue + "-" + port)
                            .namespace(deploymentNamespace).labels(labelsMap).build());
                    Ingress ingress = new Ingress();
                    ingress.setNamespace(deploymentNamespace);
                    ingress.setName(labelValue + "-" + port);
                    ingress.setLabels(labelsMap);
                    if (configContent.getIngressAnnotations() != null && configContent.getIngressAnnotations().get(port) != null) {
                        ingress.setAnnotations(configContent.getIngressAnnotations().get(port));
                    }
                    String ingressHost = configContent.getHostPrefix() + "-" + port + "." + deploymentHost;
                    if (configContent.getIngressPort() != null) {
                        if (port.equals(configContent.getIngressPort())) {
                            ingressHost = configContent.getHostPrefix() + "." + deploymentHost;
                        }
                    } else {
                        if (ports.length == 1 || port.equals("80")) {
                            ingressHost = configContent.getHostPrefix() + "." + deploymentHost;
                        }
                    }
                    ingress.setAnnotation(Ingress.REWRITE_ANNOTATION, Ingress.DEFAULT_REWRITE_PATH);
                    ingress.setHttpRules(Collections.singletonList(
                            Ingress.IngressRule.builder().host(ingressHost).ruleValues(
                                    Collections.singleton(Ingress.IngressPath.builder()
                                            .path("/")
                                            .serviceName(labelValue)
                                            .port(Integer.parseInt(port))
                                            .build())
                            ).build()
                    ));
                    if (CollectionUtils.isEmpty(ingresses) || ingresses.size() != 1) {
                        ingressApi.create(ingress);
                    } else {
                        ingressApi.update(ingress);
                    }
                }
            }
            // service
            List<Service> svcs = serviceApi.getByCondition(K8sObjectRequest.builder().namespace(deploymentNamespace).labels(labelsMap).build());
            Service svc = Service.builder().name(labelValue).labels(labelsMap).namespace(deploymentNamespace).selectors(labelsMap).servicePorts(
                    Arrays.stream(ports).map(port ->
                            ServicePort.builder()
                                    .name(labelValue + "-" + port)
                                    .port(Integer.parseInt(port))
                                    .targetPort(Integer.parseInt(port))
                                    .build()).collect(Collectors.toSet())).build();
            if (CollectionUtils.isEmpty(svcs) || svcs.size() != 1) {
                serviceApi.create(svc);
            } else {
                serviceApi.update(svc);
            }
            // 检查secret，没有则创建
            SecretVO secretVO = secretApi.getSecretByName(deploymentNamespace, DEFAULT_IMAGE_PULL_SECRET_NAME);
            if (secretVO == null) {
                secretApi.createPrivateRegistrySecret(
                        deploymentNamespace,
                        DEFAULT_IMAGE_PULL_SECRET_NAME,
                        applicationProperties.getK8s().getImageRegistry(),
                        applicationProperties.getDocker().getRegistryUsername(),
                        applicationProperties.getDocker().getRegistryPassword()
                );
            }
            // configMap
            List<ConfigMap> configMaps = configMapApi.getByCondition(K8sObjectRequest.builder().namespace(deploymentNamespace).labels(labelsMap).build());
            ConfigMap configMap = ConfigMap.builder().data(configContent.getMountsConfigMaps()).build();
            configMap.setName(labelValue);
            configMap.setNamespace(deploymentNamespace);
            configMap.setLabel(labelKey, labelValue);
            if (CollectionUtils.isEmpty(configMaps) || configMaps.size() != 1) {
                configMapApi.create(configMap);
            } else {
                configMapApi.update(configMap);
            }
            // deployment
            List<Deployment> deployments = deploymentApi.getByCondition(K8sObjectRequest.builder().namespace(deploymentNamespace).labels(labelsMap).build());
            List<V1Volume> volumes = new ArrayList<>();
            List<Container.VolumeMount> volumeMounts = new ArrayList<>();
            if (!CollectionUtils.isEmpty(configMap.getData())) {
                volumes.add(new V1VolumeBuilder().withName(labelValue).withConfigMap(
                        new V1ConfigMapVolumeSourceBuilder().withName(labelValue).withItems(
                                configMap.getData() == null ? null :
                                        configMap.getData().keySet().stream()
                                                .map(path -> {
                                                    String relativePath = path.replaceAll(ConfigVO.ConfigContent.PATH_DELIMITER, "/");
                                                    String[] arrs = relativePath.split("/");
                                                    return new V1KeyToPathBuilder()
                                                            .withKey(path)
                                                            .withPath(
                                                                    arrs[arrs.length - 1]
                                                            )
                                                            .build();
                                                })
                                                .collect(Collectors.toList())
                        ).build()
                ).build());
                volumeMounts.addAll(configMap.getData().keySet().stream()
                        .map(path -> {
                            String relativePath = path.replaceAll(ConfigVO.ConfigContent.PATH_DELIMITER, "/");
                            String[] arrs = relativePath.split("/");
                            return Container.VolumeMount.builder()
                                    .name(labelValue)
                                    .mountPath(relativePath)
                                    .subPath(arrs[arrs.length - 1])
                                    .build();
                        })
                        .collect(Collectors.toList()));
            }
            if (!CollectionUtils.isEmpty(configContent.getHostPath())) {
                for (Map.Entry<String, String> entry : configContent.getHostPath().entrySet()) {
                    String name = labelValue + "-" + entry.getKey().replaceAll("[/.]", "-");
                    volumes.add(new V1VolumeBuilder().withName(name).withHostPath(
                            new V1HostPathVolumeSourceBuilder().withPath(entry.getKey()).build()
                    ).build());
                    volumeMounts.add(Container.VolumeMount.builder()
                            .name(name)
                            .mountPath(entry.getValue())
                            .build());
                }
            }

            if (!CollectionUtils.isEmpty(configContent.getPvcPaths())) {
                for (Map.Entry<String, String> entry : configContent.getPvcPaths().entrySet()) {
                    String name = labelValue + "-" + entry.getKey();
                    volumes.add(new V1VolumeBuilder().withName(name).withPersistentVolumeClaim(
                            new V1PersistentVolumeClaimVolumeSourceBuilder().withClaimName("a" + result.getAppId() + "-" + entry.getKey()).build()
                    ).build());
                    volumeMounts.add(Container.VolumeMount.builder()
                            .name(name)
                            .mountPath(entry.getValue())
                            .build());
                }
            }
            Deployment deployment = Deployment.builder()
                    .imagePullSecrets(Collections.singletonList(DEFAULT_IMAGE_PULL_SECRET_NAME))
                    .timeout(Deployment.DEFAULT_PROGRESS_DEADLINE_SECONDS)
                    .revisionHistoryLimit(Deployment.DEFAULT_REVISION_HISTORY_LIMIT)
                    .replicas(configContent.getReplicas() == null ? 1 : configContent.getReplicas())
                    .nodeSelectors(configContent.getNodeIp() == null ? null : Collections.singletonMap("paas.seecoder.cn/ip", configContent.getNodeIp()))
                    .volumes(volumes)
                    .containers(Collections.singletonList(
                            Container.builder()
                                    .env(configContent.getEnvs())
                                    .ports(
                                            Arrays.stream(ports).map(port ->
                                                    ContainerPort.builder()
                                                            .name(application.getId() + "a" + result.getId() + "e" + port)
                                                            .port(Integer.parseInt(port))
                                                            .protocol("TCP")
                                                            .build()).collect(Collectors.toSet())
                                    )
                                    .args(StringUtils.isEmpty(configContent.getRunArgs()) ? null : Arrays.asList(configContent.getRunArgs().split(" ")))
                                    .command(StringUtils.isEmpty(configContent.getRunCommands()) ? null : Arrays.asList(configContent.getRunCommands().split(" ")))
                                    .name(labelValue)
                                    .image(imageName)
                                    .volumeMounts(volumeMounts)
                                    .build()))
                    .build();
            deployment.setName(labelValue);
            deployment.setNamespace(deploymentNamespace);
            deployment.setLabel(labelKey, labelValue);
            if (CollectionUtils.isEmpty(deployments) || deployments.size() != 1) {
                deploymentApi.create(deployment);
            } else {
                deploymentApi.update(deployment);
            }
        } catch (Exception e) {
            if (result.getBuildStatus() == BuildStatus.BUILDING) {
                result.setBuildStatus(BuildStatus.FAIL);
                result.setBuildOutput(e.getLocalizedMessage());
            }
            if (result.getDeployStatus() == DeployStatus.DEPLOYING) {
                result.setDeployStatus(DeployStatus.FAIL);
                result.setDeployOutput(e.getLocalizedMessage());
            }
            environmentDAO.save(result);
            LoggerUtil.error(logger, e, "流程失败！");
            return;
        }
    }
}