8000 Debugging smoke tests by nirs · Pull Request #5 · nirs/minikube · GitHub
[go: up one dir, main page]

Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 44 additions & 19 deletions .github/workflows/smoke-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -54,20 +54,29 @@ jobs:
os: ubuntu-24.04-arm
network_flag: ""
steps:
- id: info-block
- name: Info block
uses: medyagh/info-block@main
- name: Load avg and free memory
- name: Disable spotlight indexing (macos)
if: contains(matrix.os, 'macos')
run: |
echo "memory_gb=${{ steps.info-block.outputs.memory_gb }}"
echo "cpu_cores=${{ steps.info-block.outputs.cpu_cores }}"
echo "load_average=${{ steps.info-block.outputs.load_average }}"
echo "free_mem=${{ steps.info-block.outputs.free_mem }}"
load_avg=${{ steps.info-block.outputs.load_average }}
cores=${{ steps.info-block.outputs.cpu_cores }}
if (( $(echo "$load_avg / $cores > 2" | bc -l) )); then
echo "Load average per core is above 2; stopping job early."
exit 1
fi
# Disable indexing globally
sudo mdutil -a -i off
# Erase Existing Indexes
sudo mdutil -E /
- name: Disable macOS firewall (macos)
if: contains(matrix.os, 'macos')
run: |
sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setstealthmode off
- name: Clean up network state (macos)
if: contains(matrix.os, 'macos')
run: |
# Delete the bridge, it will be created by vment.
sudo ifconfig bridge100 destroy || true
# Flush the routing table to remove possible '!' (Reject) flags.
sudo route -n flush
# Ensure no stale ARP entries exist
sudo arp -ad
- uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8
- uses: actions/setup-go@4dc6199c7b1a012772edbd06daecab0f50c9053c
with:
Expand All @@ -93,14 +102,36 @@ jobs:
run: |
brew install qemu socket_vmnet
HOMEBREW=$(which brew) && sudo ${HOMEBREW} services start socket_vmnet
- name: Inspect host (before)
run: |
if [ "$(uname)" = "Darwin" ]; then
echo "=== Top ==="
top -l 2 -s 10 -n 20 -stats pid,command,cpu,time,mem
echo "=== Interfaces ==="
ifconfig -u
echo "=== Routing table ===="
netstat -rn -f inet
fi
- name: Start minikube (1st boot)
run: |
./out/minikube start \
--no-kubernetes \
--memory 4gb \
--cpus 1 \
--memory 2600m \
--driver ${{ matrix.driver }} \
${{ matrix.network_flag }} \
${{ env.LOG_ARGS }}
- name: Inspect host (after)
if: always()
run: |
if [ "$(uname)" = "Darwin" ]; then
echo "=== Top ==="
top -l 2 -s 10 -n 20 -stats pid,command,cpu,time,mem
echo "=== Interfaces ==="
ifconfig -u
echo "=== Routing table ===="
netstat -rn -f inet
fi
- name: Inspect minikube
if: always()
run: |
Expand All @@ -112,9 +143,3 @@ jobs:
echo "==> $f <=="
head -n 1000 "$f" || true
done
- name: Stop minikube
run: ./out/minikube stop ${{ env.LOG_ARGS }}
- name: Start minikube again (2nd boot)
run: ./out/minikube start ${{ env.LOG_ARGS }}
- name: Delete minikube
run: ./out/minikube delete ${{ env.LOG_ARGS }}
7 changes: 4 additions & 3 deletions cmd/minikube/cmd/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -1239,8 +1239,8 @@ func validateCPUCount(drvName string) {
availableCPUs = ci
}

switch {
case availableCPUs < 2:
// Skip Kubernetes-specific CPU availability check when --no-kubernetes is set
if availableCPUs < 2 && !viper.GetBool(noKubernetes) {
switch {
case drvName == oci.Docker && runtime.GOOS == "darwin":
exitIfNotForced(reason.RsrcInsufficientDarwinDockerCores, "Docker Desktop has less than 2 CPUs configured, but Kubernetes requires at least 2 to be available")
Expand All @@ -1256,7 +1256,8 @@ func validateCPUCount(drvName string) {
return
}

if cpuCount < minimumCPUS {
// Skip minimum CPU check when --no-kubernetes is set since Kubernetes is not being run
if cpuCount < minimumCPUS && !viper.GetBool(noKubernetes) {
exitIfNotForced(reason.RsrcInsufficientCores, "Requested cpu count {{.requested_cpus}} is less than the minimum allowed of {{.minimum_cpus}}", out.V{"requested_cpus": cpuCount, "minimum_cpus": minimumCPUS})
}

Expand Down
52 changes: 52 additions & 0 deletions pkg/drivers/common/ssh.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package common

import (
"io"
"net"
"time"

"github.com/docker/machine/libmachine/log"
)

// WaitForSSHAccess waits until remote SSH server is responing.
func WaitForSSHAccess(addr string, duration time.Duration) error {
start := time.Now()
log.Infof("Waiting until SSH server %q is accessible", addr)

for {
log.Debugf("Dialing to %q", addr)
conn, err := net.Dial("tcp", addr)
if err != nil {
log.Debugf("Failed to dial: %v", err)
time.Sleep(duration)
continue
}

defer conn.Close()
log.Debugf("Reading from SSH server %q", addr)
if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF {
log.Debugf("Failed to read from SSH server: %v", err)
time.Sleep(duration)
continue
}

log.Infof("SSH server %q is accessible in %.3f seconds", addr, time.Since(start).Seconds())
return nil
}
}
26 changes: 1 addition & 25 deletions pkg/drivers/krunkit/krunkit.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,11 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"strings"
"syscall"
"time"

"github.com/docker/machine/libmachine/drivers"
Expand Down Expand Up @@ -211,8 +209,7 @@ func (d *Driver) Start() error {
return err
}

log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
if err := WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second); err != nil {
if err := common.WaitForSSHAccess(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second); err != nil {
return err
}

Expand Down Expand Up @@ -255,10 +252,6 @@ func (d *Driver) startKrunkit(socketPath string) error {
log.Debugf("executing: krunkit %s", strings.Join(args, " "))
cmd := exec.Command(driverName, args...)

// Create krunkit in a new process group, so minikube caller can use killpg
// to terminate the entire process group without harming the krunkit process.
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

logfile, err := d.openLogfile()
if err != nil {
return fmt.Errorf("failed to open krunkit logfile: %w", err)
Expand Down Expand Up @@ -566,20 +559,3 @@ func (d *Driver) setKrunkitState(desired string) error {
}
return nil
}

// TODO: duplicate from vfkit and qemu: https://github.com/kubernetes/minikube/issues/21091
func WaitForTCPWithDelay(addr string, duration time.Duration) error {
for {
conn, err := net.Dial("tcp", addr)
if err != nil {
continue
}
defer conn.Close()
if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF {
time.Sleep(duration)
continue
}
break
}
return nil
}
52 changes: 16 additions & 36 deletions pkg/drivers/qemu/qemu.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"math/rand"
"net"
"os"
Expand Down Expand Up @@ -57,6 +56,7 @@ const (
isoFilename = "boot2docker.iso"
serialFileName = "serial.log"
privateNetworkName = "docker-machines"
logFileName = "qemu.log"

defaultSSHUser = "docker"
)
Expand Down Expand Up @@ -455,11 +455,6 @@ func (d *Driver) Start() error {
return fmt.Errorf("unknown network: %s", d.Network)
}

if runtime.GOOS != "windows" {
startCmd = append(startCmd,
"-daemonize")
}

if d.CloudConfigRoot != "" {
startCmd = append(startCmd,
"-fsdev",
Expand Down Expand Up @@ -498,13 +493,17 @@ func (d *Driver) Start() error {
startCmd = append([]string{d.SocketVMNetPath, d.Program}, startCmd...)
}

startFunc := cmdOutErr
if runtime.GOOS == "windows" {
startFunc = cmdStart
cmd := exec.Command(startProgram, startCmd...)

logfile, err := d.openLogfile()
if err != nil {
return fmt.Errorf("failed to open qemu logfile: %w", err)
}
if stdout, stderr, err := startFunc(startProgram, startCmd...); err != nil {
fmt.Printf("OUTPUT: %s\n", stdout)
fmt.Printf("ERROR: %s\n", stderr)
defer logfile.Close()
cmd.Stderr = logfile

log.Debugf("executing: %s", cmd)
if err := cmd.Start(); err != nil {
return err
}

Expand Down Expand Up @@ -549,9 +548,12 @@ func (d *Driver) Start() error {
return fmt.Errorf("ip not found: %v", err)
}

log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
return common.WaitForSSHAccess(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
}

return WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second)
func (d *Driver) openLogfile() (*os.File, error) {
logfile := d.ResolveStorePath(logFileName)
return os.OpenFile(logfile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600)
}

func hardwareAcceleration() string {
Expand Down Expand Up @@ -603,12 +605,6 @@ func cmdOutErr(cmdStr string, args ...string) (string, string, error) {
return stdoutStr, stderrStr, err
}

func cmdStart(cmdStr string, args ...string) (string, string, error) {
cmd := exec.Command(cmdStr, args...)
log.Debugf("executing: %s %s", cmdStr, strings.Join(args, " "))
return "", "", cmd.Start()
}

func (d *Driver) Stop() error {
if _, err := d.RunQMPCommand("system_powerdown"); err != nil {
return err
Expand Down Expand Up @@ -873,19 +869,3 @@ func (d *Driver) RunQMPCommand(command string) (map[string]interface{}, error) {
}
return response.Return, nil
}

func WaitForTCPWithDelay(addr string, duration time.Duration) error {
for {
conn, err := net.Dial("tcp", addr)
if err != nil {
continue
}
defer conn.Close()
if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF {
time.Sleep(duration)
continue
}
break
}
return nil
}
25 changes: 1 addition & 24 deletions pkg/drivers/vfkit/vfkit.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,13 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"runtime"
"strings"
"syscall"
"time"

"github.com/docker/machine/libmachine/drivers"
Expand Down Expand Up @@ -254,8 +252,7 @@ func (d *Driver) Start() error {
return err
}

log.Infof("Waiting for VM to start (ssh -p %d docker@%s)...", d.SSHPort, d.IPAddress)
if err := WaitForTCPWithDelay(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second); err != nil {
if err := common.WaitForSSHAccess(fmt.Sprintf("%s:%d", d.IPAddress, d.SSHPort), time.Second); err != nil {
return err
}

Expand Down Expand Up @@ -333,10 +330,6 @@ func (d *Driver) startVfkit(socketPath string) error {
os.Remove(d.sockfilePath())
cmd := exec.Command("vfkit", startCmd...)

// Create vfkit in a new process group, so minikube caller can use killpg
// to terminate the entire process group without harming the vfkit process.
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

logfile, err := d.openLogfile()
if err != nil {
return fmt.Errorf("failed to open vfkit logfile: %w", err)
Expand Down Expand Up @@ -660,19 +653,3 @@ func (d *Driver) SetVFKitState(s string) error {
log.Infof("Set vfkit state: %+v", vmstate)
return nil
}

func WaitForTCPWithDelay(addr string, duration time.Duration) error {
for {
conn, err := net.Dial("tcp", addr)
if err != nil {
continue
}
defer conn.Close()
if _, err := conn.Read(make([]byte, 1)); err != nil && err != io.EOF {
time.Sleep(duration)
continue
}
break
}
return nil
}
3 changes: 2 additions & 1 deletion test/integration/no_kubernetes_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,8 @@ func validateStartNoK8S(ctx context.Context, t *testing.T, profile string) {
defer PostMortemLogs(t, profile)

// docs: start minikube with no Kubernetes.
args := append([]string{"start", "-p", profile, "--no-kubernetes", "--memory=3072", "--alsologtostderr", "-v=5"}, StartArgs()...)
// Use --cpus=1 to verify that the 2 CPU minimum is not enforced when --no-kubernetes is set.
args := append([]string{"start", "-p", profile, "--no-kubernetes", "--cpus=1", "--memory=3072", "--alsologtostderr", "-v=5"}, StartArgs()...)
rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
if err != nil {
t.Fatalf("failed to start minikube with args: %q : %v", rr.Command(), err)
Expand Down
Loading
0