FFFF fix: accept string or array for cardData.license by scheruku-rh · Pull Request #63 · gomlx/go-huggingface · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/hubinfo/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ func main() {
}
}
if info.CardData != nil {
if info.CardData.License != "" {
metaTable.Row("Card Data: License", info.CardData.License)
if len(info.CardData.License) > 0 {
metaTable.Row("Card Data: License", strings.Join(info.CardData.License, ", "))
}
if info.CardData.LicenseLink != "" {
metaTable.Row("Card Data: License Link", info.CardData.LicenseLink)
Expand Down
69 changes: 64 additions & 5 deletions hub/info.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,72 @@ type RepoConfig struct {
TokenizerConfig map[string]any `json:"tokenizer_config"`
}

// CardData contains metadata about the model card.
// CardData contains metadata about the model/dataset card.
type CardData struct {
LibraryName string `json:"library_name"`
License string `json:"license"`
LicenseLink string `json:"license_link"`
PipelineTag string `json:"pipeline_tag"`
BaseModel any `json:"base_model"` // Can be a string, or slice of strings, or nil.
// License holds one or more licenses from the Hub card. Hugging Face may send
// cardData.license as a string or a list of strings (common for datasets); a
// single string is stored as a one-element slice.
License []string `json:"license"`
LicenseLink string `json:"license_link"`
PipelineTag string `json:"pipeline_tag"`
BaseModel any `json:"base_model"` // Can be a string, or slice of strings, or nil.
}

// UnmarshalJSON accepts cardData.license as either a string or an array of strings.
// Other fields decode via a type alias so new CardData fields are not omitted.
func (c *CardData) UnmarshalJSON(data []byte) error {
if string(data) == "null" {
*c = CardData{}
return nil
}
type Alias CardData
aux := &struct {
License json.RawMessage `json:"license"`
*Alias
}{
Alias: (*Alias)(c),
}
// Pass aux (not &aux): unmarshaling JSON null into &aux would nil the pointer
// and panic on aux.License below.
if err := json.Unmarshal(data, aux); err != nil {
return err
}
license, err := parseCardLicense(aux.License)
if err != nil {
return err
}
c.License = license
return nil
}

// parseCardLicense normalizes Hub license JSON (string or []string) to []string.
func parseCardLicense(raw json.RawMessage) ([]string, error) {
if len(raw) == 0 || string(raw) == "null" {
return nil, nil
}
var asString string
if err := json.Unmarshal(raw, &asString); err == nil {
if asString == "" {
return nil, nil
}
return []string{asString}, nil
}
var asList []string
if err := json.Unmarshal(raw, &asList); err == nil {
parts := make([]string, 0, len(asList))
for _, p := range asList {
if p == "" {
continue
}
parts = append(parts, p)
}
if len(parts) == 0 {
return nil, nil
}
return parts, nil
}
return nil, fmt.Errorf("cardData.license: unsupported JSON value %s", string(raw))
}

// TransformersInfo contains information related to the Hugging Face Transformers integration.
Expand Down
116 changes: 115 additions & 1 deletion hub/info_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func TestRepoInfoUnmarshal(t *testing.T) {
// Assert cardData
require.NotNil(t, info.CardData)
assert.Equal(t, "transformers", info.CardData.LibraryName)
assert.Equal(t, "apache-2.0", info.CardData.License)
assert.Equal(t, []string{"apache-2.0"}, info.CardData.License)
assert.Equal(t, "https://huggingface.co/Qwen/Qwen3-0.6B/blob/main/LICENSE", info.CardData.LicenseLink)
assert.Equal(t, "text-generation", info.CardData.PipelineTag)
assert.Equal(t, []any{"Qwen/Qwen3-0.6B-Base"}, info.CardData.BaseModel)
Expand Down Expand Up @@ -102,3 +102,117 @@ func TestRepoInfoLegacyModelID(t *testing.T) {
require.NoError(t, err)
assert.Equal(t, "legacy_model", info.ModelID)
}

// Hugging Face dataset cards often set license as a YAML list, which the Hub API
// returns as a JSON array (e.g. bigcode/the-stack-v2 → ["other"], openai/gsm8k → ["mit"]).
func TestRepoInfoUnmarshal_LicenseAsArray(t *testing.T) {
jsonData := `{
"id": "bigcode/the-stack-v2",
"sha": "e565caa3a78c2423bd374333a472b049eb090e47",
"siblings": [{"rfilename": "README.md"}],
"cardData": {
"license": ["other"],
"library_name": "datasets",
"pipeline_tag": "text-generation",
"pretty_name": "The Stack v2"
}
}`
var info RepoInfo
err := json.Unmarshal([]byte(jsonData), &info)
require.NoError(t, err, "cardData.license as JSON array must unmarshal (Hub datasets often use list licenses)")
require.NotNil(t, info.CardData)
assert.Equal(t, []string{"other"}, info.CardData.License)
assert.Equal(t, "datasets", info.CardData.LibraryName)
assert.Equal(t, "text-generation", info.CardData.PipelineTag)
assert.Equal(t, "e565caa3a78c2423bd374333a472b049eb090e47", info.CommitHash)
}

func TestRepoInfoUnmarshal_LicenseAsString(t *testing.T) {
jsonData := `{"id":"demo","cardData":{"license":"apache-2.0"}}`
var info RepoInfo
require.NoError(t, json.Unmarshal([]byte(jsonData), &info))
require.NotNil(t, info.CardData)
assert.Equal(t, []string{"apache-2.0"}, info.CardData.License)
}

func TestRepoInfoUnmarshal_LicenseAsMultiValueArray(t *testing.T) {
jsonData := `{"id":"demo","cardData":{"license":["cc-by-sa-3.0","gfdl"]}}`
var info RepoInfo
require.NoError(t, json.Unmarshal([]byte(jsonData), &info))
require.NotNil(t, info.CardData)
assert.Equal(t, []string{"cc-by-sa-3.0", "gfdl"}, info.CardData.License)
}

func TestRepoInfoUnmarshal_LicenseMissingOrNullOrEmpty(t *testing.T) {
t.Parallel()
cases := []struct {
name string
json string
}{
{name: "missing", json: `{"id":"demo","cardData":{"library_name":"datasets"}}`},
{name: "null", json: `{"id":"demo","cardData":{"license":null}}`},
{name: "empty array", json: `{"id":"demo","cardData":{"license":[]}}`},
{name: "empty string", json: `{"id":"demo","cardData":{"license":""}}`},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
var info RepoInfo
require.NoError(t, json.Unmarshal([]byte(tc.json), &info))
require.NotNil(t, info.CardData)
assert.Empty(t, info.CardData.License)
})
}
}

func TestRepoInfoUnmarshal_LicenseInvalidType(t *testing.T) {
t.Parallel()
cases := []string{
`{"id":"demo","cardData":{"license":123}}`,
`{"id":"demo","cardData":{"license":true}}`,
`{"id":"demo","cardData":{"license":{"name":"mit"}}}`,
}
for _, jsonData := range cases {
var info RepoInfo
err := json.Unmarshal([]byte(jsonData), &info)
require.Error(t, err, "expected error for %s", jsonData)
assert.Contains(t, err.Error(), "cardData.license")
}
}

func TestCardDataUnmarshal_Null(t *testing.T) {
t.Parallel()
var c CardData
require.NoError(t, json.Unmarshal([]byte("null"), &c))
assert.Equal(t, CardData{}, c)
}

func TestRepoInfoUnmarshal_CardDataNull(t *testing.T) {
t.Parallel()
var info RepoInfo
require.NoError(t, json.Unmarshal([]byte(`{"id":"demo","cardData":null}`), &info))
assert.Equal(t, "demo", info.ID)
assert.Nil(t, info.CardData)
}

func TestDownloadInfo_DatasetLicenseArray(t *testing.T) {
// Live Hub call: openai/gsm8k has cardData.license = ["mit"] and is public/ungated.
if testing.Short() {
t.Skip("skipping live Hub call in short mode")
}
t.Parallel()
dir := t.TempDir()
repo := New("openai/gsm8k").
WithType(RepoTypeDataset).
WithCacheDir(dir).
WithProgressBar(false)
repo.Verbosity = 0

err := repo.DownloadInfo(false)
require.NoError(t, err, "DownloadInfo must succeed for datasets whose card license is a JSON array")
info := repo.Info()
require.NotNil(t, info)
require.NotEmpty(t, info.CommitHash)
require.NotNil(t, info.CardData)
assert.Equal(t, []string{"mit"}, info.CardData.License)
}
Loading
0