/*
Copyright 2020 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"
	"net"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/blang/semver/v4"
	"github.com/docker/machine/libmachine"
	"github.com/docker/machine/libmachine/host"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"golang.org/x/sync/errgroup"
	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/klog/v2"
	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
	"k8s.io/minikube/pkg/addons"
	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/pkg/minikube/bootstrapper"
	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
	"k8s.io/minikube/pkg/minikube/cluster"
	"k8s.io/minikube/pkg/minikube/cni"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/cruntime"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/kubeconfig"
	"k8s.io/minikube/pkg/minikube/localpath"
	"k8s.io/minikube/pkg/minikube/logs"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/mustload"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/out/register"
	"k8s.io/minikube/pkg/minikube/proxy"
	"k8s.io/minikube/pkg/minikube/reason"
	"k8s.io/minikube/pkg/minikube/registry"
	"k8s.io/minikube/pkg/minikube/style"
	"k8s.io/minikube/pkg/minikube/vmpath"
	"k8s.io/minikube/pkg/util"
	"k8s.io/minikube/pkg/util/retry"
	kconst "k8s.io/minikube/third_party/kubeadm/app/constants"
)

const waitTimeout = "wait-timeout"

var (
	kicGroup   errgroup.Group
	cacheGroup errgroup.Group
)

// Starter is a struct with all the necessary information to start a node
type Starter struct {
	Runner         command.Runner
	PreExists      bool
	StopK8s        bool
	MachineAPI     libmachine.API
	Host           *host.Host
	Cfg            *config.ClusterConfig
	Node           *config.Node
	ExistingAddons map[string]bool
}

// Start spins up a guest and starts the Kubernetes node.
func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) {
	var wg sync.WaitGroup
	stopk8s, err := handleNoKubernetes(starter)
	if err != nil {
		return nil, err
	}
	if stopk8s {
		nv := semver.Version{Major: 0, Minor: 0, Patch: 0}
		cr := configureRuntimes(starter.Runner, *starter.Cfg, nv)

		showNoK8sVersionInfo(cr)

		configureMounts(&wg, *starter.Cfg)
		return nil, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
	}

	// wait for preloaded tarball to finish downloading before configuring runtimes
	waitCacheRequiredImages(&cacheGroup)

	sv, err := util.ParseKubernetesVersion(starter.Node.KubernetesVersion)
	if err != nil {
		return nil, errors.Wrap(err, "Failed to parse Kubernetes version")
	}

	// configure the runtime (docker, containerd, crio)
	cr := configureRuntimes(starter.Runner, *starter.Cfg, sv)

	// check if installed runtime is compatible with current minikube code
	if err = cruntime.CheckCompatibility(cr); err != nil {
		return nil, err
	}

	showVersionInfo(starter.Node.KubernetesVersion, cr)

	// Add "host.minikube.internal" DNS alias (intentionally non-fatal)
	hostIP, err := cluster.HostIP(starter.Host, starter.Cfg.Name)
	if err != nil {
		klog.Errorf("Unable to get host IP: %v", err)
	} else if err := machine.AddHostAlias(starter.Runner, constants.HostAlias, hostIP); err != nil {
		klog.Errorf("Unable to add host alias: %v", err)
	}

	var kcs *kubeconfig.Settings
	var bs bootstrapper.Bootstrapper
	if apiServer {
		kcs, bs, err = handleAPIServer(starter, cr, hostIP)
		if err != nil {
			return nil, err
		}
	} else {
		bs, err = cluster.Bootstrapper(starter.MachineAPI, viper.GetString(cmdcfg.Bootstrapper), *starter.Cfg, starter.Runner)
		if err != nil {
			return nil, errors.Wrap(err, "Failed to get bootstrapper")
		}

		if err = bs.SetupCerts(*starter.Cfg, *starter.Node); err != nil {
			return nil, errors.Wrap(err, "setting up certs")
		}

		if err := bs.UpdateNode(*starter.Cfg, *starter.Node, cr); err != nil {
			return nil, errors.Wrap(err, "update node")
		}
	}

	go configureMounts(&wg, *starter.Cfg)

	wg.Add(1)
	go func() {
		defer wg.Done()
		profile, err := config.LoadProfile(starter.Cfg.Name)
		if err != nil {
			out.FailureT("Unable to load profile: {{.error}}", out.V{"error": err})
		}
		if err := CacheAndLoadImagesInConfig([]*config.Profile{profile}); err != nil {
			out.FailureT("Unable to push cached images: {{.error}}", out.V{"error": err})
		}
	}()

	// enable addons, both old and new!
	addonList := viper.GetStringSlice(config.AddonListFlag)
	if starter.ExistingAddons != nil {
		if viper.GetBool("force") {
			addons.Force = true
		}
		wg.Add(1)
		go addons.Start(&wg, starter.Cfg, starter.ExistingAddons, addonList)
	}

	// discourage use of the virtualbox driver
	if starter.Cfg.Driver == driver.VirtualBox && viper.GetBool(config.WantVirtualBoxDriverWarning) {
		warnVirtualBox()
	}

	if apiServer {
		// special ops for none , like change minikube directory.
		// multinode super doesn't work on the none driver
		if starter.Cfg.Driver == driver.None && len(starter.Cfg.Nodes) == 1 {
			prepareNone()
		}
	} else {
		// Make sure to use the command runner for the control plane to generate the join token
		cpBs, cpr, err := cluster.ControlPlaneBootstrapper(starter.MachineAPI, starter.Cfg, viper.GetString(cmdcfg.Bootstrapper))
		if err != nil {
			return nil, errors.Wrap(err, "getting control plane bootstrapper")
		}

		if err := joinCluster(starter, cpBs, bs); err != nil {
			return nil, errors.Wrap(err, "joining cp")
		}

		cnm, err := cni.New(starter.Cfg)
		if err != nil {
			return nil, errors.Wrap(err, "cni")
		}

		if err := cnm.Apply(cpr); err != nil {
			return nil, errors.Wrap(err, "cni apply")
		}
	}
	klog.Infof("Will wait %s for node %+v", viper.GetDuration(waitTimeout), starter.Node)
	if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil {
		return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout))
	}

	klog.Infof("waiting for startup goroutines ...")
	wg.Wait()

	// Write enabled addons to the config before completion
	return kcs, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
}

// handleNoKubernetes handles starting minikube without Kubernetes.
func handleNoKubernetes(starter Starter) (bool, error) {
	// Do not bootstrap cluster if --no-kubernetes.
	if starter.Node.KubernetesVersion == constants.NoKubernetesVersion {
		// Stop existing Kubernetes node if applicable.
		if starter.StopK8s {
			cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket})
			if err != nil {
				return false, err
			}
			kubeadm.StopKubernetes(starter.Runner, cr)
		}
		return true, config.Write(viper.GetString(config.ProfileName), starter.Cfg)
	}
	return false, nil
}

// handleAPIServer handles starting the API server.
func handleAPIServer(starter Starter, cr cruntime.Manager, hostIP net.IP) (*kubeconfig.Settings, bootstrapper.Bootstrapper, error) {
	var err error

	// Must be written before bootstrap, otherwise health checks may flake due to stale IP.
	kcs := setupKubeconfig(starter.Host, starter.Cfg, starter.Node, starter.Cfg.Name)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Failed to setup kubeconfig")
	}

	// Setup kubeadm (must come after setupKubeconfig).
	bs := setupKubeAdm(starter.MachineAPI, *starter.Cfg, *starter.Node, starter.Runner)
	err = bs.StartCluster(*starter.Cfg)
	if err != nil {
		ExitIfFatal(err)
		out.LogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, *starter.Cfg, starter.Runner))
		return nil, bs, err
	}

	// Write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper.
	if err := kubeconfig.Update(kcs); err != nil {
		return nil, bs, errors.Wrap(err, "Failed kubeconfig update")
	}

	if !starter.Cfg.DisableOptimizations {
		// Scale down CoreDNS from default 2 to 1 replica.
		if err := kapi.ScaleDeployment(starter.Cfg.Name, meta.NamespaceSystem, kconst.CoreDNSDeploymentName, 1); err != nil {
			klog.Errorf("Unable to scale down deployment %q in namespace %q to 1 replica: %v", kconst.CoreDNSDeploymentName, meta.NamespaceSystem, err)
		}
	}

	// Not running this in a Go func can result in DNS answering taking up to 38 seconds, with the Go func it takes 6-10 seconds.
	go func() {
		// Inject {"host.minikube.internal": hostIP} record into CoreDNS.
		if err := addCoreDNSEntry(starter.Runner, "host.minikube.internal", hostIP.String(), *starter.Cfg); err != nil {
			klog.Warningf("Unable to inject {%q: %s} record into CoreDNS: %v", "host.minikube.internal", hostIP.String(), err)
			out.Err("Failed to inject host.minikube.internal into CoreDNS, this will limit the pods access to the host IP")
		}
	}()
	return kcs, bs, nil
}

// joinCluster adds new or prepares and then adds existing node to the cluster.
func joinCluster(starter Starter, cpBs bootstrapper.Bootstrapper, bs bootstrapper.Bootstrapper) error {
	start := time.Now()
	klog.Infof("JoinCluster: %+v", starter.Cfg)
	defer func() {
		klog.Infof("JoinCluster complete in %s", time.Since(start))
	}()

	joinCmd, err := cpBs.GenerateToken(*starter.Cfg)
	if err != nil {
		return fmt.Errorf("error generating join token: %w", err)
	}

	// avoid "error execution phase kubelet-start: a Node with name "<name>" and status "Ready" already exists in the cluster.
	// You must delete the existing Node or change the name of this new joining Node"
	if starter.PreExists {
		klog.Infof("removing existing worker node %q before attempting to rejoin cluster: %+v", starter.Node.Name, starter.Node)
		if _, err := drainNode(*starter.Cfg, starter.Node.Name); err != nil {
			klog.Errorf("error removing existing worker node before rejoining cluster, will continue anyway: %v", err)
		}
		klog.Infof("successfully removed existing worker node %q from cluster: %+v", starter.Node.Name, starter.Node)
	}

	join := func() error {
		klog.Infof("trying to join worker node %q to cluster: %+v", starter.Node.Name, starter.Node)
		if err := bs.JoinCluster(*starter.Cfg, *starter.Node, joinCmd); err != nil {
			klog.Errorf("worker node failed to join cluster, will retry: %v", err)

			// reset worker node to revert any changes made by previous kubeadm init/join
			klog.Infof("resetting worker node %q before attempting to rejoin cluster...", starter.Node.Name)
			if _, err := starter.Runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(starter.Cfg.KubernetesConfig.KubernetesVersion)))); err != nil {
				klog.Infof("kubeadm reset failed, continuing anyway: %v", err)
			} else {
				klog.Infof("successfully reset worker node %q", starter.Node.Name)
			}

			return err
		}
		return nil
	}
	if err := retry.Expo(join, 10*time.Second, 3*time.Minute); err != nil {
		return fmt.Errorf("error joining worker node to cluster: %w", err)
	}

	return nil
}

// Provision provisions the machine/container for the node
func Provision(cc *config.ClusterConfig, n *config.Node, apiServer bool, delOnFail bool) (command.Runner, bool, libmachine.API, *host.Host, error) {
	register.Reg.SetStep(register.StartingNode)
	name := config.MachineName(*cc, *n)

	// Be explicit with each case for the sake of translations
	if cc.KubernetesConfig.KubernetesVersion == constants.NoKubernetesVersion {
		out.Step(style.ThumbsUp, "Starting minikube without Kubernetes in cluster {{.cluster}}", out.V{"cluster": cc.Name})
	} else {
		if apiServer {
			out.Step(style.ThumbsUp, "Starting control plane node {{.name}} in cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
		} else {
			out.Step(style.ThumbsUp, "Starting worker node {{.name}} in cluster {{.cluster}}", out.V{"name": name, "cluster": cc.Name})
		}

	}

	if driver.IsKIC(cc.Driver) {
		beginDownloadKicBaseImage(&kicGroup, cc, viper.GetBool("download-only"))
	}

	if !driver.BareMetal(cc.Driver) {
		beginCacheKubernetesImages(&cacheGroup, cc.KubernetesConfig.ImageRepository, n.KubernetesVersion, cc.KubernetesConfig.ContainerRuntime, cc.Driver)
	}

	// Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot.
	// Hence, SaveProfile must be called before startHost, and again afterwards when we know the IP.
	if err := config.SaveProfile(viper.GetString(config.ProfileName), cc); err != nil {
		return nil, false, nil, nil, errors.Wrap(err, "Failed to save config")
	}

	handleDownloadOnly(&cacheGroup, &kicGroup, n.KubernetesVersion, cc.KubernetesConfig.ContainerRuntime, cc.Driver)
	waitDownloadKicBaseImage(&kicGroup)

	return startMachine(cc, n, delOnFail)
}

// ConfigureRuntimes does what needs to happen to get a runtime going.
func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, kv semver.Version) cruntime.Manager {
	co := cruntime.Config{
		Type:              cc.KubernetesConfig.ContainerRuntime,
		Socket:            cc.KubernetesConfig.CRISocket,
		Runner:            runner,
		ImageRepository:   cc.KubernetesConfig.ImageRepository,
		KubernetesVersion: kv,
		InsecureRegistry:  cc.InsecureRegistry,
	}
	cr, err := cruntime.New(co)
	if err != nil {
		exit.Error(reason.InternalRuntime, "Failed runtime", err)
	}

	disableOthers := true
	if driver.BareMetal(cc.Driver) {
		disableOthers = false
	}

	// Preload is overly invasive for bare metal, and caching is not meaningful.
	// KIC handles preload elsewhere.
	if driver.IsVM(cc.Driver) {
		if err := cr.Preload(cc); err != nil {
			switch err.(type) {
			case *cruntime.ErrISOFeature:
				out.ErrT(style.Tip, "Existing disk is missing new features ({{.error}}). To upgrade, run 'minikube delete'", out.V{"error": err})
			default:
				klog.Warningf("%s preload failed: %v, falling back to caching images", cr.Name(), err)
			}

			if err := machine.CacheImagesForBootstrapper(cc.KubernetesConfig.ImageRepository, cc.KubernetesConfig.KubernetesVersion, viper.GetString(cmdcfg.Bootstrapper)); err != nil {
				exit.Error(reason.RuntimeCache, "Failed to cache images", err)
			}
		}
	}

	if kv.GTE(semver.MustParse("1.24.0-alpha.2")) {
		if err := cruntime.ConfigureNetworkPlugin(cr, runner, cc.KubernetesConfig.NetworkPlugin); err != nil {
			exit.Error(reason.RuntimeEnable, "Failed to configure network plugin", err)
		}
	}

	inUserNamespace := strings.Contains(cc.KubernetesConfig.FeatureGates, "KubeletInUserNamespace=true")
	err = cr.Enable(disableOthers, forceSystemd(), inUserNamespace)
	if err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to enable container runtime", err)
	}

	// Wait for the CRI to be "live", before returning it
	err = waitForCRISocket(runner, cr.SocketPath(), 60, 1)
	if err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to start container runtime", err)
	}

	// Wait for the CRI to actually work, before returning
	err = waitForCRIVersion(runner, cr.SocketPath(), 60, 10)
	if err != nil {
		exit.Error(reason.RuntimeEnable, "Failed to start container runtime", err)
	}
	return cr
}

func forceSystemd() bool {
	return viper.GetBool("force-systemd") || os.Getenv(constants.MinikubeForceSystemdEnv) == "true"
}

func pathExists(runner cruntime.CommandRunner, path string) (bool, error) {
	_, err := runner.RunCmd(exec.Command("stat", path))
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func waitForCRISocket(runner cruntime.CommandRunner, socket string, wait int, interval int) error {

	if socket == "" || socket == "/var/run/dockershim.sock" {
		return nil
	}

	klog.Infof("Will wait %ds for socket path %s", wait, socket)

	chkPath := func() error {
		e, err := pathExists(runner, socket)
		if err != nil {
			return err
		}
		if !e {
			return &retry.RetriableError{Err: err}
		}
		return nil
	}
	return retry.Expo(chkPath, time.Duration(interval)*time.Second, time.Duration(wait)*time.Second)
}

func waitForCRIVersion(runner cruntime.CommandRunner, socket string, wait int, interval int) error {

	if socket == "" || socket == "/var/run/dockershim.sock" {
		return nil
	}

	klog.Infof("Will wait %ds for crictl version", wait)

	chkInfo := func() error {
		args := []string{"crictl", "version"}
		cmd := exec.Command("sudo", args...)
		rr, err := runner.RunCmd(cmd)
		if err != nil && !os.IsNotExist(err) {
			return &retry.RetriableError{Err: err}
		}
		klog.Info(rr.Stdout.String())
		return nil
	}
	return retry.Expo(chkInfo, time.Duration(interval)*time.Second, time.Duration(wait)*time.Second)
}

// setupKubeAdm adds any requested files into the VM before Kubernetes is started
func setupKubeAdm(mAPI libmachine.API, cfg config.ClusterConfig, n config.Node, r command.Runner) bootstrapper.Bootstrapper {
	bs, err := cluster.Bootstrapper(mAPI, viper.GetString(cmdcfg.Bootstrapper), cfg, r)
	if err != nil {
		exit.Error(reason.InternalBootstrapper, "Failed to get bootstrapper", err)
	}
	for _, eo := range cfg.KubernetesConfig.ExtraOptions {
		out.Infof("{{.extra_option_component_name}}.{{.key}}={{.value}}", out.V{"extra_option_component_name": eo.Component, "key": eo.Key, "value": eo.Value})
	}
	// Loads cached images, generates config files, download binaries
	// update cluster and set up certs

	if err := bs.UpdateCluster(cfg); err != nil {
		if errors.Is(err, cruntime.ErrContainerRuntimeNotRunning) {
			exit.Error(reason.KubernetesInstallFailedRuntimeNotRunning, "Failed to update cluster", err)
		}
		exit.Error(reason.KubernetesInstallFailed, "Failed to update cluster", err)
	}

	if err := bs.SetupCerts(cfg, n); err != nil {
		exit.Error(reason.GuestCert, "Failed to setup certs", err)
	}

	return bs
}

func setupKubeconfig(h *host.Host, cc *config.ClusterConfig, n *config.Node, clusterName string) *kubeconfig.Settings {
	addr, err := apiServerURL(*h, *cc, *n)
	if err != nil {
		exit.Message(reason.DrvCPEndpoint, fmt.Sprintf("failed to get API Server URL: %v", err), out.V{"profileArg": fmt.Sprintf("--profile=%s", clusterName)})
	}

	if cc.KubernetesConfig.APIServerName != constants.APIServerName {
		addr = strings.ReplaceAll(addr, n.IP, cc.KubernetesConfig.APIServerName)
	}
	kcs := &kubeconfig.Settings{
		ClusterName:          clusterName,
		Namespace:            cc.KubernetesConfig.Namespace,
		ClusterServerAddress: addr,
		ClientCertificate:    localpath.ClientCert(cc.Name),
		ClientKey:            localpath.ClientKey(cc.Name),
		CertificateAuthority: localpath.CACert(),
		KeepContext:          cc.KeepContext,
		EmbedCerts:           cc.EmbedCerts,
	}

	kcs.SetPath(kubeconfig.PathFromEnv())
	return kcs
}

func apiServerURL(h host.Host, cc config.ClusterConfig, n config.Node) (string, error) {
	hostname, _, port, err := driver.ControlPlaneEndpoint(&cc, &n, h.DriverName)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("https://" + net.JoinHostPort(hostname, strconv.Itoa(port))), nil
}

// StartMachine starts a VM
func startMachine(cfg *config.ClusterConfig, node *config.Node, delOnFail bool) (runner command.Runner, preExists bool, machineAPI libmachine.API, host *host.Host, err error) {
	m, err := machine.NewAPIClient()
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to get machine client")
	}
	host, preExists, err = startHostInternal(m, cfg, node, delOnFail)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to start host")
	}
	runner, err = machine.CommandRunner(host)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to get command runner")
	}

	ip, err := validateNetwork(host, runner, cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
	if err != nil {
		return runner, preExists, m, host, errors.Wrap(err, "Failed to validate network")
	}

	if driver.IsQEMU(host.Driver.DriverName()) {
		apiServerPort, err := getPort()
		if err != nil {
			return runner, preExists, m, host, errors.Wrap(err, "Failed to find apiserver port")
		}
		cfg.APIServerPort = apiServerPort
	}

	// Bypass proxy for minikube's vm host ip
	err = proxy.ExcludeIP(ip)
	if err != nil {
		out.FailureT("Failed to set NO_PROXY Env. Please use `export NO_PROXY=$NO_PROXY,{{.ip}}`.", out.V{"ip": ip})
	}

	return runner, preExists, m, host, err
}

// getPort asks the kernel for a free open port that is ready to use
func getPort() (int, error) {
	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	if err != nil {
		panic(err)
	}

	l, err := net.ListenTCP("tcp", addr)
	if err != nil {
		return -1, errors.Errorf("Error accessing port %d", addr.Port)
	}
	defer l.Close()
	return l.Addr().(*net.TCPAddr).Port, nil
}

// startHostInternal starts a new minikube host using a VM or None
func startHostInternal(api libmachine.API, cc *config.ClusterConfig, n *config.Node, delOnFail bool) (*host.Host, bool, error) {
	host, exists, err := machine.StartHost(api, cc, n)
	if err == nil {
		return host, exists, nil
	}
	klog.Warningf("error starting host: %v", err)
	// NOTE: People get very cranky if you delete their preexisting VM. Only delete new ones.
	if !exists {
		err := machine.DeleteHost(api, config.MachineName(*cc, *n))
		if err != nil {
			klog.Warningf("delete host: %v", err)
		}
	}

	if err, ff := errors.Cause(err).(*oci.FailFastError); ff {
		klog.Infof("will skip retrying to create machine because error is not retriable: %v", err)
		return host, exists, err
	}

	out.ErrT(style.Embarrassed, "StartHost failed, but will try again: {{.error}}", out.V{"error": err})
	klog.Info("Will try again in 5 seconds ...")
	// Try again, but just once to avoid making the logs overly confusing
	time.Sleep(5 * time.Second)

	if delOnFail {
		klog.Info("Deleting existing host since delete-on-failure was set.")
		// Delete the failed existing host
		err := machine.DeleteHost(api, config.MachineName(*cc, *n))
		if err != nil {
			klog.Warningf("delete host: %v", err)
		}
	}

	host, exists, err = machine.StartHost(api, cc, n)
	if err == nil {
		return host, exists, nil
	}

	// Don't use host.Driver to avoid nil pointer deref
	drv := cc.Driver
	out.ErrT(style.Sad, `Failed to start {{.driver}} {{.driver_type}}. Running "{{.cmd}}" may fix it: {{.error}}`, out.V{"driver": drv, "driver_type": driver.MachineType(drv), "cmd": mustload.ExampleCmd(cc.Name, "delete"), "error": err})
	return host, exists, err
}

// validateNetwork tries to catch network problems as soon as possible
func validateNetwork(h *host.Host, r command.Runner, imageRepository string, kubernetesVersion string) (string, error) {
	ip, err := h.Driver.GetIP()
	if err != nil {
		return ip, err
	}

	optSeen := false
	warnedOnce := false
	for _, k := range proxy.EnvVars {
		if v := os.Getenv(k); v != "" {
			if !optSeen {
				out.Styled(style.Internet, "Found network options:")
				optSeen = true
			}
			out.Infof("{{.key}}={{.value}}", out.V{"key": k, "value": v})
			ipExcluded := proxy.IsIPExcluded(ip) // Skip warning if minikube ip is already in NO_PROXY
			k = strings.ToUpper(k)               // for http_proxy & https_proxy
			if (k == "HTTP_PROXY" || k == "HTTPS_PROXY") && !ipExcluded && !warnedOnce {
				out.WarningT("You appear to be using a proxy, but your NO_PROXY environment does not include the minikube IP ({{.ip_address}}).", out.V{"ip_address": ip})
				out.Styled(style.Documentation, "Please see {{.documentation_url}} for more details", out.V{"documentation_url": "https://minikube.sigs.k8s.io/docs/handbook/vpn_and_proxy/"})
				warnedOnce = true
			}
		}
	}

	if !driver.BareMetal(h.Driver.DriverName()) && !driver.IsKIC(h.Driver.DriverName()) && !driver.IsQEMU(h.Driver.DriverName()) {
		if err := trySSH(h, ip); err != nil {
			return ip, err
		}
	}

	// Non-blocking
	go tryRegistry(r, h.Driver.DriverName(), imageRepository, kubernetesVersion)
	return ip, nil
}

func trySSH(h *host.Host, ip string) error {
	if viper.GetBool("force") {
		return nil
	}

	sshAddr := net.JoinHostPort(ip, "22")

	dial := func() (err error) {
		d := net.Dialer{Timeout: 3 * time.Second}
		conn, err := d.Dial("tcp", sshAddr)
		if err != nil {
			klog.Warningf("dial failed (will retry): %v", err)
			return err
		}
		_ = conn.Close()
		return nil
	}

	err := retry.Expo(dial, time.Second, 13*time.Second)
	if err != nil {
		out.ErrT(style.Failure, `minikube is unable to connect to the VM: {{.error}}

	This is likely due to one of two reasons:

	- VPN or firewall interference
	- {{.hypervisor}} network configuration issue

	Suggested workarounds:

	- Disable your local VPN or firewall software
	- Configure your local VPN or firewall to allow access to {{.ip}}
	- Restart or reinstall {{.hypervisor}}
	- Use an alternative --vm-driver
	- Use --force to override this connectivity check
	`, out.V{"error": err, "hypervisor": h.Driver.DriverName(), "ip": ip})
	}

	return err
}

// tryRegistry tries to connect to the image repository
func tryRegistry(r command.Runner, driverName string, imageRepository string, kubernetesVersion string) {
	// 2 second timeout. For best results, call tryRegistry in a non-blocking manner.
	opts := []string{"-sS", "-m", "2"}

	proxy := os.Getenv("HTTPS_PROXY")
	if proxy != "" && !strings.HasPrefix(proxy, "localhost") && !strings.HasPrefix(proxy, "127.0") {
		opts = append([]string{"-x", proxy}, opts...)
	}

	if imageRepository == "" {
		v, _ := util.ParseKubernetesVersion(kubernetesVersion)
		imageRepository = images.DefaultKubernetesRepo(v)
	}

	opts = append(opts, fmt.Sprintf("https://%s/", imageRepository))
	if rr, err := r.RunCmd(exec.Command("curl", opts...)); err != nil {
		klog.Warningf("%s failed: %v", rr.Args, err)
		out.WarningT("This {{.type}} is having trouble accessing https://{{.repository}}", out.V{"repository": imageRepository, "type": driver.MachineType(driverName)})
		out.ErrT(style.Tip, "To pull new external images, you may need to configure a proxy: https://minikube.sigs.k8s.io/docs/reference/networking/proxy/")
	}
}

// prepareNone prepares the user and host for the joy of the "none" driver
func prepareNone() {
	register.Reg.SetStep(register.ConfiguringLHEnv)
	out.Step(style.StartingNone, "Configuring local host environment ...")
	if viper.GetBool(config.WantNoneDriverWarning) {
		out.ErrT(style.Empty, "")
		out.WarningT("The 'none' driver is designed for experts who need to integrate with an existing VM")
		out.ErrT(style.Tip, "Most users should use the newer 'docker' driver instead, which does not require root!")
		out.ErrT(style.Documentation, "For more information, see: https://minikube.sigs.k8s.io/docs/reference/drivers/none/")
		out.ErrT(style.Empty, "")
	}

	if os.Getenv("CHANGE_MINIKUBE_NONE_USER") == "" {
		home := os.Getenv("HOME")
		out.WarningT("kubectl and minikube configuration will be stored in {{.home_folder}}", out.V{"home_folder": home})
		out.WarningT("To use kubectl or minikube commands as your own user, you may need to relocate them. For example, to overwrite your own settings, run:")

		out.ErrT(style.Empty, "")
		out.ErrT(style.Command, "sudo mv {{.home_folder}}/.kube {{.home_folder}}/.minikube $HOME", out.V{"home_folder": home})
		out.ErrT(style.Command, "sudo chown -R $USER $HOME/.kube $HOME/.minikube")
		out.ErrT(style.Empty, "")

		out.ErrT(style.Tip, "This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_USER=true")
	}

	if err := util.MaybeChownDirRecursiveToMinikubeUser(localpath.MiniPath()); err != nil {
		exit.Message(reason.HostHomeChown, "Failed to change permissions for {{.minikube_dir_path}}: {{.error}}", out.V{"minikube_dir_path": localpath.MiniPath(), "error": err})
	}
}

// addCoreDNSEntry adds host name and IP record to the DNS by updating CoreDNS's ConfigMap.
// ref: https://coredns.io/plugins/hosts/
// note: there can be only one 'hosts' block in CoreDNS's ConfigMap (avoid "plugin/hosts: this plugin can only be used once per Server Block" error)
func addCoreDNSEntry(runner command.Runner, name, ip string, cc config.ClusterConfig) error {
	kubectl := kapi.KubectlBinaryPath(cc.KubernetesConfig.KubernetesVersion)
	kubecfg := path.Join(vmpath.GuestPersistentDir, "kubeconfig")

	// get current coredns configmap via kubectl
	get := fmt.Sprintf("sudo %s --kubeconfig=%s -n kube-system get configmap coredns -o yaml", kubectl, kubecfg)
	out, err := runner.RunCmd(exec.Command("/bin/bash", "-c", get))
	if err != nil {
		klog.Errorf("failed to get current CoreDNS ConfigMap: %v", err)
		return err
	}
	cm := strings.TrimSpace(out.Stdout.String())

	// check if this specific host entry already exists in coredns configmap, so not to duplicate/override it
	host := regexp.MustCompile(fmt.Sprintf(`(?smU)^ *hosts {.*%s.*}`, name))
	if host.MatchString(cm) {
		klog.Infof("CoreDNS already contains %q host record, skipping...", name)
		return nil
	}

	// inject hosts block with host record into coredns configmap
	sed := fmt.Sprintf("sed '/^        forward . \\/etc\\/resolv.conf.*/i \\        hosts {\\n           %s %s\\n           fallthrough\\n        }'", ip, name)
	// check if hosts block already exists in coredns configmap
	hosts := regexp.MustCompile(`(?smU)^ *hosts {.*}`)
	if hosts.MatchString(cm) {
		// inject host record into existing coredns configmap hosts block instead
		klog.Info("CoreDNS already contains hosts block, will inject host record there...")
		sed = fmt.Sprintf("sed '/^        hosts {.*/a \\           %s %s'", ip, name)
	}

	// replace coredns configmap via kubectl
	replace := fmt.Sprintf("sudo %s --kubeconfig=%s replace -f -", kubectl, kubecfg)
	if _, err := runner.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s | %s | %s", get, sed, replace))); err != nil {
		klog.Errorf("failed to inject {%q: %s} host record into CoreDNS", name, ip)
		return err
	}
	klog.Infof("{%q: %s} host record injected into CoreDNS", name, ip)

	return nil
}

// prints a warning to the console against the use of the 'virtualbox' driver, if alternatives are available and healthy
func warnVirtualBox() {
	var altDriverList strings.Builder
	for _, choice := range driver.Choices(true) {
		if choice.Name != "virtualbox" && choice.Priority != registry.Discouraged && choice.State.Installed && choice.State.Healthy {
			altDriverList.WriteString(fmt.Sprintf("\n\t- %s", choice.Name))
		}
	}

	if altDriverList.Len() != 0 {
		out.Boxed(`You have selected "virtualbox" driver, but there are better options !
For better performance and support consider using a different driver: {{.drivers}}

To turn off this warning run:

	$ minikube config set WantVirtualBoxDriverWarning false


To learn more about on minikube drivers checkout https://minikube.sigs.k8s.io/docs/drivers/
To see benchmarks checkout https://minikube.sigs.k8s.io/docs/benchmarks/cpuusage/

`, out.V{"drivers": altDriverList.String()})
	}
}
