Repository navigation
<
8000
div class="prc-PageLayout-PageLayoutContent-BneH9"> More file actions
Expand file tree
/
Copy pathhub.go
More file actions
116 lines (102 loc) · 3.63 KB
/
Copy pathhub.go
File metadata and controls
116 lines (102 loc) · 3.63 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
// Package hub can be used to download and cache files from HuggingFace Hub, which may
// be models, tokenizers or anything.
//
// It is meant to be a port of huggingFace_hub python library to Go, and be able to share the same
// cache structure (usually under "~/.cache/huggingface/hub").
//
// It is also safe to be used concurrently by multiple programs -- it uses file system lock to control concurrency.
//
// Typical usage will be something like:
//
// repo := hub.New(modelID).WithAuth(hfAuthToken)
// var fileNames []string
// for fileName, err := range repo.IterFileNames() {
// if err != nil { panic(err) }
// fmt.Printf("\t%s\n", fileName)
// fileNames = append(fileNames, fileName)
// }
// downloadedFiles, err := repo.DownloadFiles(fileNames...)
// if err != nil { ... }
//
// From here, downloadedFiles will point to files in the local cache that one can read.
//
// If the model/dataset files are already available in a plain local directory (e.g. from `git clone` or
// `huggingface-cli download --local-dir`), use NewLocal instead of New to read them directly, without any
// network access:
//
// repo := hub.NewLocal("/path/to/local/model/dir")
//
// A Repo created with NewLocal is a drop-in replacement: every function in this module that takes a *Repo
// works the same way, whether it is backed by the HuggingFace cache or by a local directory.
//
// Environment variables:
//
// - HF_ENDPOINT: Where to connect to huggingface, default is https://huggingface.co
// - XDG_CACHE_HOME: Cache directory, defaults to ${HOME}/.cache
package hub
import (
"fmt"
"os"
"path"
"runtime"
"strings"
"github.com/gomlx/go-huggingface"
"github.com/google/uuid"
"github.com/pkg/errors"
)
// SessionId is unique and always created anew at the start of the program, and used during the life of the program.
var SessionId string
// panicf generates an error message and panics with it, in one function.
func panicf(format string, args ...any) {
err := errors.Errorf(format, args...)
panic(err)
}
func init() {
sessionUUID, err := uuid.NewRandom()
if err != nil {
panicf("failed generating UUID for SessionId: %v", err)
}
SessionId = strings.Replace(sessionUUID.String(), "-", "", -1)
}
var (
// DefaultDirCreationPerm is used when creating new cache subdirectories.
DefaultDirCreationPerm = os.FileMode(0755)
// DefaultFileCreationPerm is used when creating files inside the cache subdirectories.
DefaultFileCreationPerm = os.FileMode(0644)
)
const (
HeaderXRepoCommit = "X-Repo-Commit"
HeaderXLinkedETag = "X-Linked-Etag"
HeaderXLinkedSize = "X-Linked-Size"
)
func getEnvOr(key, defaultValue string) string {
v := os.Getenv(key)
if v == "" {
return defaultValue
}
return v
}
// DefaultCacheDir for HuggingFace Hub, same used by the python library.
//
// Its prefix is either `${XDG_CACHE_HOME}` if set, or `~/.cache` otherwise. Followed by `/huggingface/hub/`.
// So typically: `~/.cache/huggingface/hub/`.
func DefaultCacheDir() string {
cacheDir := getEnvOr("XDG_CACHE_HOME", path.Join(os.Getenv("HOME"), ".cache"))
cacheDir = path.Join(cacheDir, "huggingface", "hub")
return cacheDir
}
// DefaultHttpUserAgent returns a user agent to use with HuggingFace Hub API.
func DefaultHttpUserAgent() string {
return fmt.Sprintf("go-huggingface/%v; golang/%s; session_id/%s",
huggingface.Version, runtime.Version(), SessionId)
}
// RepoIdSeparator is used to separate repository/model names parts when mapping to file names.
// Likely only for internal use.
const RepoIdSeparator = "--"
// RepoType supported by HuggingFace-Hub
type RepoType string
const (
RepoTypeDataset RepoType = "datasets"
RepoTypeSpace RepoType = "spaces"
RepoTypeModel RepoType = "models"
)