8000 minio/cmd/admin-handlers.go at master · ksea/minio · GitHub
[go: up one dir, main page]
Skip to content

admin-handlers.go

Latest commit

 

History

History
3563 lines (3131 loc) · 99.7 KB

admin-handlers.go

File metadata and controls

3563 lines (3131 loc) · 99.7 KB
Raw
// Copyright (c) 2015-2022 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"bytes"
"context"
crand "crypto/rand"
"crypto/rsa"
"crypto/subtle"
"crypto/x509"
"encoding/base64"
"encoding/json"
"encoding/pem"
"errors"
"fmt"
"hash/crc32"
"io"
"math"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/dustin/go-humanize"
"github.com/klauspost/compress/zip"
"github.com/minio/madmin-go/v3"
"github.com/minio/madmin-go/v3/estream"
"github.com/minio/minio-go/v7/pkg/set"
"github.com/minio/minio/internal/auth"
"github.com/minio/minio/internal/dsync"
"github.com/minio/minio/internal/grid"
"github.com/minio/minio/internal/handlers"
xhttp "github.com/minio/minio/internal/http"
xioutil "github.com/minio/minio/internal/ioutil"
"github.com/minio/minio/internal/kms"
"github.com/minio/minio/internal/logger"
"github.com/minio/mux"
"github.com/minio/pkg/v3/logger/message/log"
xnet "github.com/minio/pkg/v3/net"
"github.com/minio/pkg/v3/policy"
"github.com/secure-io/sio-go"
"github.com/zeebo/xxh3"
)
const (
maxEConfigJSONSize = 262272
kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
anonymizeParam = "anonymize"
anonymizeStrict = "strict"
)
// Only valid query params for mgmt admin APIs.
const (
mgmtBucket = "bucket"
mgmtPrefix = "prefix"
mgmtClientToken = "clientToken"
mgmtForceStart = "forceStart"
mgmtForceStop = "forceStop"
)
// ServerUpdateV2Handler - POST /minio/admin/v3/update?updateURL={updateURL}&type=2
// ----------
// updates all minio servers and restarts them gracefully.
func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
if objectAPI == nil {
return
}
if globalInplaceUpdateDisabled {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
return
}
if currentReleaseTime.IsZero() || currentReleaseTime.Equal(timeSentinel) {
apiErr := errorCodes.ToAPIErr(ErrMethodNotAllowed)
apiErr.Description = fmt.Sprintf("unable to perform in-place update, release time is unrecognized: %s", currentReleaseTime)
writeErrorResponseJSON(ctx, w, apiErr, r.URL)
return
}
vars := mux.Vars(r)
updateURL := vars["updateURL"]
dryRun := r.Form.Get("dry-run") == "true"
mode := getMinioMode()
if updateURL == "" {
updateURL = minioReleaseInfoURL
if runtime.GOOS == globalWindowsOSName {
updateURL = minioReleaseWindowsInfoURL
}
}
local := globalLocalNodeName
if local == "" {
local = "127.0.0.1"
}
u, err := url.Parse(updateURL)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
content, err := downloadReleaseURL(u, updateTimeout, mode)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
updateStatus := madmin.ServerUpdateStatusV2{
DryRun: dryRun,
Results: make([]madmin.ServerPeerUpdateStatus, 0, len(globalNotificationSys.allPeerClients)),
}
peerResults := make(map[string]madmin.ServerPeerUpdateStatus, len(globalNotificationSys.allPeerClients))
failedClients := make(map[int]bool, len(globalNotificationSys.allPeerClients))
if lrTime.Sub(currentReleaseTime) <= 0 {
updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
Host: local,
Err: fmt.Sprintf("server is running the latest version: %s", Version),
CurrentVersion: Version,
})
for _, client := range globalNotificationSys.peerClients {
updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{
Host: client.String(),
Err: fmt.Sprintf("server is running the latest version: %s", Version),
CurrentVersion: Version,
})
}
// Marshal API response
jsonBytes, err := json.Marshal(updateStatus)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, jsonBytes)
return
}
u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
// Download Binary Once
binC, bin, err := downloadBinary(u, mode)
if err != nil {
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if globalIsDistErasure {
// Push binary to other servers
for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
if nerr.Err != nil {
peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
Host: nerr.Host.String(),
Err: nerr.Err.Error(),
CurrentVersion: Version,
}
failedClients[idx] = true
} else {
peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
Host: nerr.Host.String(),
CurrentVersion: Version,
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
}
}
}
}
if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
peerResults[local] = madmin.ServerPeerUpdateStatus{
Host: local,
Err: err.Error(),
CurrentVersion: Version,
}
} else {
peerResults[local] = madmin.ServerPeerUpdateStatus{
Host: local,
CurrentVersion: Version,
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
}
}
if !dryRun {
if globalIsDistErasure {
ng := WithNPeers(len(globalNotificationSys.peerClients))
for idx, client := range globalNotificationSys.peerClients {
if failedClients[idx] {
continue
}
client := client
ng.Go(ctx, func() error {
return client.CommitBinary(ctx)
}, idx, *client.host)
}
for _, nerr := range ng.Wait() {
if nerr.Err != nil {
prs, ok := peerResults[nerr.Host.String()]
if ok {
prs.Err = nerr.Err.Error()
peerResults[nerr.Host.String()] = prs
} else {
peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
Host: nerr.Host.String(),
Err: nerr.Err.Error(),
CurrentVersion: Version,
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
}
}
}
}
}
prs := peerResults[local]
if prs.Err == "" {
if err = commitBinary(); err != nil {
prs.Err = err.Error()
}
peerResults[local] = prs
}
}
prs, ok := peerResults[local]
if ok {
prs.WaitingDrives = waitingDrivesNode()
peerResults[local] = prs
}
if globalIsDistErasure {
// Notify all other MinIO peers signal service.
startTime := time.Now().Add(restartUpdateDelay)
ng := WithNPeers(len(globalNotificationSys.peerClients))
for idx, client := range globalNotificationSys.peerClients {
if failedClients[idx] {
continue
}
client := client
ng.Go(ctx, func() error {
prs, ok := peerResults[client.String()]
// We restart only on success, not for any failures.
if ok && prs.Err == "" {
return client.SignalService(serviceRestart, "", dryRun, &startTime)
}
return nil
}, idx, *client.host)
}
for _, nerr := range ng.Wait() {
if nerr.Err != nil {
waitingDrives := map[string]madmin.DiskMetrics{}
jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
if jerr == nil {
prs, ok := peerResults[nerr.Host.String()]
if ok {
prs.WaitingDrives = waitingDrives
peerResults[nerr.Host.String()] = prs
}
continue
}
}
}
}
for _, pr := range peerResults {
updateStatus.Results = append(updateStatus.Results, pr)
}
// Marshal API response
jsonBytes, err := json.Marshal(updateStatus)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, jsonBytes)
if !dryRun {
prs, ok := peerResults[local]
// We restart only on success, not for any failures.
if ok && prs.Err == "" {
globalServiceSignalCh <- serviceRestart
}
}
}
// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
// ----------
// updates all minio servers and restarts them gracefully.
func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
if objectAPI == nil {
return
}
if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
return
}
vars := mux.Vars(r)
updateURL := vars["updateURL"]
mode := getMinioMode()
if updateURL == "" {
updateURL = minioReleaseInfoURL
if runtime.GOOS == globalWindowsOSName {
updateURL = minioReleaseWindowsInfoURL
}
}
u, err := url.Parse(updateURL)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
content, err := downloadReleaseURL(u, updateTimeout, mode)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
if lrTime.Sub(currentReleaseTime) <= 0 {
updateStatus := madmin.ServerUpdateStatus{
CurrentVersion: Version,
UpdatedVersion: Version,
}
// Marshal API response
jsonBytes, err := json.Marshal(updateStatus)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, jsonBytes)
return
}
u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
// Download Binary Once
binC, bin, err := downloadBinary(u, mode)
if err != nil {
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Push binary to other servers
for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
if nerr.Err != nil {
err := AdminError{
Code: AdminUpdateApplyFailure,
Message: nerr.Err.Error(),
StatusCode: http.StatusInternalServerError,
}
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin))
if err != nil {
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
for _, nerr := range globalNotificationSys.CommitBinary(ctx) {
if nerr.Err != nil {
err := AdminError{
Code: AdminUpdateApplyFailure,
Message: nerr.Err.Error(),
StatusCode: http.StatusInternalServerError,
}
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
err = commitBinary()
if err != nil {
adminLogIf(ctx, fmt.Errorf("server update failed with %w", err))
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
updateStatus := madmin.ServerUpdateStatus{
CurrentVersion: Version,
UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout),
}
// Marshal API response
jsonBytes, err := json.Marshal(updateStatus)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
writeSuccessResponseJSON(w, jsonBytes)
// Notify all other MinIO peers signal service.
for _, nerr := range globalNotificationSys.SignalService(serviceRestart) {
if nerr.Err != nil {
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
adminLogIf(ctx, nerr.Err)
}
}
globalServiceSignalCh <- serviceRestart
}
// ServiceHandler - POST /minio/admin/v3/service?action={action}
// ----------
// Supports following actions:
// - restart (restarts all the MinIO instances in a setup)
// - stop (stops all the MinIO instances in a setup)
// - freeze (freezes all incoming S3 API calls)
// - unfreeze (unfreezes previously frozen S3 API calls)
func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
action := vars["action"]
var serviceSig serviceSignal
switch madmin.ServiceAction(action) {
case madmin.ServiceActionRestart:
serviceSig = serviceRestart
case madmin.ServiceActionStop:
serviceSig = serviceStop
case madmin.ServiceActionFreeze:
serviceSig = serviceFreeze
case madmin.ServiceActionUnfreeze:
serviceSig = serviceUnFreeze
default:
adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
return
}
var objectAPI ObjectLayer
switch serviceSig {
case serviceRestart:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
case serviceStop:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
case serviceFreeze, serviceUnFreeze:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
}
if objectAPI == nil {
return
}
// Notify all other MinIO peers signal service.
for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
if nerr.Err != nil {
logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
adminLogIf(ctx, nerr.Err)
}
}
// Reply to the client before restarting, stopping MinIO server.
writeSuccessResponseHeadersOnly(w)
switch serviceSig {
case serviceFreeze:
freezeServices()
case serviceUnFreeze:
unfreezeServices()
case serviceRestart, serviceStop:
globalServiceSignalCh <- serviceSig
}
}
type servicePeerResult struct {
Host string `json:"host"`
Err string `json:"err,omitempty"`
WaitingDrives map[string]madmin.DiskMetrics `json:"waitingDrives,omitempty"`
}
type serviceResult struct {
Action madmin.ServiceAction `json:"action"`
DryRun bool `json:"dryRun"`
Results []servicePeerResult `json:"results,omitempty"`
}
// ServiceV2Handler - POST /minio/admin/v3/service?action={action}&type=2
// ----------
// Supports following actions:
// - restart (restarts all the MinIO instances in a setup)
// - stop (stops all the MinIO instances in a setup)
// - freeze (freezes all incoming S3 API calls)
// - unfreeze (unfreezes previously frozen S3 API calls)
//
// This newer API now returns back status per remote peer and local regarding
// if a "restart/stop" was successful or not. Service signal now supports
// a dry-run that helps skip the nodes that may have hung drives. By default
// restart/stop will ignore the servers that are hung on drives. You can use
// 'force' param to force restart even with hung drives if needed.
func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
vars := mux.Vars(r)
action := vars["action"]
dryRun := r.Form.Get("dry-run") == "true"
var serviceSig serviceSignal
act := madmin.ServiceAction(action)
switch act {
case madmin.ServiceActionRestart:
serviceSig = serviceRestart
case madmin.ServiceActionStop:
serviceSig = serviceStop
case madmin.ServiceActionFreeze:
serviceSig = serviceFreeze
case madmin.ServiceActionUnfreeze:
serviceSig = serviceUnFreeze
default:
adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
return
}
var objectAPI ObjectLayer
var execAt *time.Time
switch serviceSig {
case serviceRestart:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
t := time.Now().Add(restartUpdateDelay)
execAt = &t
case serviceStop:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
case serviceFreeze, serviceUnFreeze:
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
}
if objectAPI == nil {
return
}
// Notify all other MinIO peers signal service.
srvResult := serviceResult{Action: act, Results: []servicePeerResult{}}
process := act == madmin.ServiceActionRestart || act == madmin.ServiceActionStop
if process {
localhost := globalLocalNodeName
if globalLocalNodeName == "" {
localhost = "127.0.0.1"
}
waitingDrives := waitingDrivesNode()
srvResult.Results = append(srvResult.Results, servicePeerResult{
Host: localhost,
WaitingDrives: waitingDrives,
})
}
if globalIsDistErasure {
for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun, execAt) {
if nerr.Err != nil && process {
waitingDrives := map[string]madmin.DiskMetrics{}
jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
if jerr == nil {
srvResult.Results = append(srvResult.Results, servicePeerResult{
Host: nerr.Host.String(),
WaitingDrives: waitingDrives,
})
continue
}
}
errStr := ""
if nerr.Err != nil {
errStr = nerr.Err.Error()
}
srvResult.Results = append(srvResult.Results, servicePeerResult{
Host: nerr.Host.String(),
Err: errStr,
})
}
}
srvResult.DryRun = dryRun
buf, err := json.Marshal(srvResult)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Reply to the client before restarting, stopping MinIO server.
writeSuccessResponseJSON(w, buf)
switch serviceSig {
case serviceFreeze:
freezeServices()
case serviceUnFreeze:
unfreezeServices()
case serviceRestart, serviceStop:
if !dryRun {
globalServiceSignalCh <- serviceSig
}
}
}
// ServerProperties holds some server information such as, version, region
// uptime, etc..
type ServerProperties struct {
Uptime int64 `json:"uptime"`
Version string `json:"version"`
CommitID string `json:"commitID"`
DeploymentID string `json:"deploymentID"`
Region string `json:"region"`
SQSARN []string `json:"sqsARN"`
}
// serverConnStats holds transferred bytes from/to the server
type serverConnStats struct {
internodeInputBytes uint64
internodeOutputBytes uint64
s3InputBytes uint64
s3OutputBytes uint64
}
// ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
// including the average duration the call was spent.
type ServerHTTPAPIStats struct {
APIStats map[string]int `json:"apiStats"`
}
// ServerHTTPStats holds all type of http operations performed to/from the server
// including their average execution time.
type ServerHTTPStats struct {
S3RequestsInQueue int32 `json:"s3RequestsInQueue"`
S3RequestsIncoming uint64 `json:"s3RequestsIncoming"`
CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"`
TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"`
TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"`
TotalS35xxErrors ServerHTTPAPIStats `json:"totalS35xxErrors"`
TotalS34xxErrors ServerHTTPAPIStats `json:"totalS34xxErrors"`
TotalS3Canceled ServerHTTPAPIStats `json:"totalS3Canceled"`
TotalS3RejectedAuth uint64 `json:"totalS3RejectedAuth"`
TotalS3RejectedTime uint64 `json:"totalS3RejectedTime"`
TotalS3RejectedHeader uint64 `json:"totalS3RejectedHeader"`
TotalS3RejectedInvalid uint64 `json:"totalS3RejectedInvalid"`
}
// StorageInfoHandler - GET /minio/admin/v3/storageinfo
// ----------
// Get server information
func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
if objectAPI == nil {
return
}
storageInfo := objectAPI.StorageInfo(ctx, true)
// Collect any disk healing.
healing, _ := getAggregatedBackgroundHealState(ctx, nil)
healDisks := make(map[string]struct{}, len(healing.HealDisks))
for _, disk := range healing.HealDisks {
healDisks[disk] = struct{}{}
}
// find all disks which belong to each respective endpoints
for i, disk := range storageInfo.Disks {
if _, ok := healDisks[disk.Endpoint]; ok {
storageInfo.Disks[i].Healing = true
}
}
// Marshal API response
jsonBytes, err := json.Marshal(storageInfo)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Reply with storage information (across nodes in a
// distributed setup) as json.
writeSuccessResponseJSON(w, jsonBytes)
}
// MetricsHandler - GET /minio/admin/v3/metrics
// ----------
// Get realtime server metrics
func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction)
if objectAPI == nil {
return
}
const defaultMetricsInterval = time.Second
interval, err := time.ParseDuration(r.Form.Get("interval"))
if err != nil || interval < time.Second {
interval = defaultMetricsInterval
}
n, err := strconv.Atoi(r.Form.Get("n"))
if err != nil || n <= 0 {
n = math.MaxInt32
}
var types madmin.MetricType
if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 {
types = madmin.MetricType(t)
} else {
types = madmin.MetricsAll
}
disks := strings.Split(r.Form.Get("disks"), ",")
byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
var diskMap map[string]struct{}
if len(disks) > 0 && disks[0] != "" {
diskMap = make(map[string]struct{}, len(disks))
for _, k := range disks {
if k != "" {
diskMap[k] = struct{}{}
}
}
}
jobID := r.Form.Get("by-jobID")
hosts := strings.Split(r.Form.Get("hosts"), ",")
byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
var hostMap map[string]struct{}
if len(hosts) > 0 && hosts[0] != "" {
hostMap = make(map[string]struct{}, len(hosts))
for _, k := range hosts {
if k != "" {
hostMap[k] = struct{}{}
}
}
}
dID := r.Form.Get("by-depID")
done := ctx.Done()
ticker := time.NewTicker(interval)
defer ticker.Stop()
w.Header().Set(xhttp.ContentType, string(mimeJSON))
enc := json.NewEncoder(w)
for n > 0 {
var m madmin.RealtimeMetrics
mLocal := collectLocalMetrics(types, collectMetricsOpts{
hosts: hostMap,
disks: diskMap,
jobID: jobID,
depID: dID,
})
m.Merge(&mLocal)
// Allow half the interval for collecting remote...
cctx, cancel := context.WithTimeout(ctx, interval/2)
mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{
hosts: hostMap,
disks: diskMap,
jobID: jobID,
depID: dID,
})
cancel()
m.Merge(&mRemote)
if !byHost {
m.ByHost = nil
}
if !byDisk {
m.ByDisk = nil
}
m.Final = n <= 1
// Marshal API reesponse
if err := enc.Encode(&m); err != nil {
n = 0
}
n--
if n <= 0 {
break
}
// Flush before waiting for next...
w.(http.Flusher).Flush()
select {
case <-ticker.C:
case <-done:
return
}
}
}
// DataUsageInfoHandler - GET /minio/admin/v3/datausage?capacity={true}
// ----------
// Get server/cluster data usage info
func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction)
if objectAPI == nil {
return
}
dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
// Get capacity info when asked.
if r.Form.Get("capacity") == "true" {
sinfo := objectAPI.StorageInfo(ctx, false)
dataUsageInfo.TotalCapacity = GetTotalUsableCapacity(sinfo.Disks, sinfo)
dataUsageInfo.TotalFreeCapacity = GetTotalUsableCapacityFree(sinfo.Disks, sinfo)
if dataUsageInfo.TotalCapacity > dataUsageInfo.TotalFreeCapacity {
dataUsageInfo.TotalUsedCapacity = dataUsageInfo.TotalCapacity - dataUsageInfo.TotalFreeCapacity
}
}
writeSuccessResponseJSON(w, dataUsageInfoJSON)
}
func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
t := time.Unix(0, l.Timestamp)
entry := &madmin.LockEntry{
Timestamp: t,
Elapsed: now.Sub(t),
Resource: resource,
ServerList: []string{server},
Source: l.Source,
Owner: l.Owner,
ID: l.UID,
Quorum: l.Quorum,
}
if l.Writer {
entry.Type = "WRITE"
} else {
entry.Type = "READ"
}
return entry
}
func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
now := time.Now().UTC()
entryMap := make(map[string]*madmin.LockEntry)
toEntry := func(lri lockRequesterInfo) string {
return fmt.Sprintf("%s/%s", lri.Name, lri.UID)
}
for _, peerLock := range peerLocks {
if peerLock == nil {
continue
}
for k, v := range peerLock.Locks {
for _, lockReqInfo := range v {
if val, ok := entryMap[toEntry(lockReqInfo)]; ok {
val.ServerList = append(val.ServerList, peerLock.Addr)
} else {
entryMap[toEntry(lockReqInfo)] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr)
}
}
}
}
var lockEntries madmin.LockEntries
for _, v := range entryMap {
if stale {
lockEntries = append(lockEntries, *v)
continue
}
if len(v.ServerList) >= v.Quorum {
lockEntries = append(lockEntries, *v)
}
}
sort.Sort(lockEntries)
return lockEntries
}
// PeerLocks holds server information result of one node
type PeerLocks struct {
Addr string
Locks map[string][]lockRequesterInfo
}
// ForceUnlockHandler force unlocks requested resource
func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction)
if objectAPI == nil {
return
}
z, ok := objectAPI.(*erasureServerPools)
if !ok {
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
return
}
vars := mux.Vars(r)
var args dsync.LockArgs
var lockers []dsync.NetLocker
for _, path := range strings.Split(vars["paths"], ",") {
if path == "" {
continue
}
args.Resources = append(args.Resources, path)
}
for _, lks := range z.serverPools[0].erasureLockers {
lockers = append(lockers, lks...)
}
for _, locker := range lockers {
locker.ForceUnlock(ctx, args)
}
}
// TopLocksHandler Get list of locks in use
func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction)
if objectAPI == nil {
return
}
count := 10 // by default list only top 10 entries
if countStr := r.Form.Get("count"); countStr != "" {
var err error
count, err = strconv.Atoi(countStr)
if err != nil {
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
return
}
}
stale := r.Form.Get("stale") == "true" // list also stale locks
peerLocks := globalNotificationSys.GetLocks(ctx, r)
topLocks := topLockEntries(peerLocks, stale)
// Marshal API response upto requested count.
if len(topLocks) > count && count > 0 {
topLocks = topLocks[:count]
}
View remainder of file in raw view
0