8000 feat(rdf): import Dublin Core descriptions into a namespace by adeelahmad · Pull Request #82 · adeelahmad/lense · GitHub
[go: up one dir, main page]

Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
8000
Diff view
19 changes: 19 additions & 0 deletions docs/rdf.md
8000
Original file line number Diff line number Diff line change
Expand Up @@ -63,10 +63,29 @@ includes those of them that refine one of its 15 elements (spatial becomes `dc:c

Both accept `?format=` or an `Accept` header, and return Turtle when neither is given.

## Import

`POST /api/v1/namespaces/{name}/rdf/import` reads Dublin Core descriptions into the namespace's recordings. It is for
editors, and the body is `{data, format, dry_run}`.

- Turtle, N-Triples and JSON-LD are accepted. A JSON-LD document must have its `@context` inline, because Lens
doesn't fetch anything while reading. RDF/XML isn't accepted. The limit is 5 MB.
- A description is matched to a recording by its URI (`/id/recording/<id>`). Failing that, it is matched by an
identifier the recording already has (`dcterms:identifier`, `dc:identifier` or `lens:lensId`). Descriptions that
match nothing are listed but not created.
- DCMI terms and the 15 DC 1.1 elements fill the fields in the mapping above. Speakers and entities named by URI must
belong to the namespace. A license that isn't Creative Commons or RightsStatements.org is kept as a statement.
- Any other statement about the recording is kept as it came (`statements`) and included again on export. Exporting a
recording and importing it back changes nothing.
- Terms and statements add to what the recording already has. Other fields replace what it has.
- `dry_run` (the default) only reports what would change. Otherwise every change is a metadata edit that is kept in
the recording's history and can be reverted, and the import is written to the audit log.

## Refine later

- A permanent vocabulary URI shared by every Lens. Today `lens:` is `<address>/ns#`, so two archives use different
terms.
- Roles qualified per recording. Today a speaker's role is left out, because the speaker is the same resource in
every recording.
- Collections and entities that visitors can see in public namespaces.
- Importing descriptions that match no recording as new resources, and importing collections, entities and speakers.
24 changes: 20 additions & 4 deletions fastapi_backend/app/api/v1/routes/rdf.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,19 @@
"""The archive as RDF (domain/rdf.py) for the app and API clients: a recording's description, or a whole namespace's
graph (its collections, recordings, entities and speakers), in Turtle, JSON-LD, N-Triples or RDF/XML."""
graph (its collections, recordings, entities and speakers), in Turtle, JSON-LD, N-Triples or RDF/XML; and Dublin Core
read back into a namespace's recordings (domain/rdf_import.py)."""

from __future__ import annotations

from typing import Any

from fastapi import APIRouter, Query, Request
from fastapi import APIRouter, HTTPException, Query, Request
from fastapi.responses import Response

from app.api.deps import Acl, Cfg, CurrentUser, Db
from app.api.deps import Acl, Cfg, CurrentUser, Db, Writer
from app.api.iiif import base_url
from app.api.linked_data import rdf_response, wanted
from app.domain import rdf
from app.domain import auth, rdf, rdf_import
from app.schemas.rdf import RdfImport, RdfImportResult

router = APIRouter(tags=["rdf"])
RDF_CONTENT: dict[int | str, dict[str, Any]] = {
Expand Down Expand Up @@ -43,3 +45,17 @@ def get_namespace_rdf(
sid = acl.namespace(name)
g = rdf.namespace_graph(db, cfg, base_url(request, cfg), sid)
return rdf_response(g, wanted(request, format) or "turtle", filename=name if download else None)


@router.post("/namespaces/{name}/rdf/import")
def import_namespace_rdf(name: str, body: RdfImport, request: Request, user: Writer, acl: Acl, db: Db, cfg: Cfg) -> RdfImportResult:
"""Read Dublin Core descriptions into the namespace's recordings (matched by their URI or an identifier). Each change
is a metadata edit, kept in the recording's history. With dry_run (the default) nothing changes."""
sid = acl.namespace(name, "editor")
try:
out = rdf_import.run(db, cfg, base_url(request, cfg), sid, body.data, body.format, body.dry_run, user.email)
except rdf_import.ImportProblem as e:
raise HTTPException(400, str(e)) from None
if not body.dry_run:
auth.audit(db, user.as_audit(), "metadata.rdf_import", f"space:{sid}", [i["recording"] for i in out["items"] if i["fields"]])
return RdfImportResult.model_validate(out)
37 changes: 37 additions & 0 deletions fastapi_backend/app/domain/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
"homepage",
"related",
"terms",
"statements",
"access",
"open",
"featured",
Expand Down Expand Up @@ -153,6 +154,40 @@ def _terms(v):
return out


STATEMENTS_MAX = 500


def _statements(v):
"""Other RDF statements about the recording, kept as they came (an RDF import: rdf.py 8000 says them again):
[{p: property URI, o: value, uri: whether the value is a resource, lang, datatype}]."""
if not isinstance(v, list):
raise MetaProblem("statements is a list of {p, o} pairs")
out = []
for x in v[:STATEMENTS_MAX]:
if not isinstance(x, dict) or not isinstance(x.get("o"), str):
raise MetaProblem("every statement has a property (p) and a value (o)")
p = _uri(x.get("p"), "a statement's property")
if not p:
raise MetaProblem("every statement has a property (p) and a value (o)")
uri = bool(x.get("uri"))
if uri:
_uri(x["o"], "a statement's value")
if x.get("lang") and not LANG_RX.match(str(x["lang"])):
raise MetaProblem(f"'{x['lang']}' isn't a language code")
st = store.clean(
{
"p": p,
"o": x["o"][:TERM_MAX],
"uri": uri or None,
"lang": None if uri else x.get("lang") or None,
"datatype": None if uri else _uri(x.get("datatype"), "a statement's datatype"),
}
)
if st not in out:
out.append(st)
return out


def clean(patch):
"""Validate and normalise fields someone is saving. None clears a field (it won't fall back to a default)."""
out = {}
Expand Down Expand Up @@ -251,6 +286,8 @@ def clean(patch):
]
elif k == "terms":
out[k] = _terms(v) or None
elif k == "statements":
out[k] = _statements(v) or None
elif k == "open":
try:
out[k] = acc.parts(v)
Expand Down
7 changes: 7 additions & 0 deletions fastapi_backend/app/domain/rdf.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,6 +189,13 @@ def add_recording(g, db, cfg, u: Uris, rid, member=False, entities=None):
for term, vals in (meta.get("terms") or {}).items():
for v in vals:
g.add((s, DCTERMS[term], _value(v)))
for st in meta.get("statements") or []:
o = (
URIRef(st["o"])
if st.get("uri")
else Literal(st["o"], lang=st.get("lang"), datatype=URIRef(st["datatype"]) if st.get("datatype") else None)
)
g.add((s, URIRef(st["p"]), o))
if row.get("duration_ms") and kind in ("audio", "video"):
g.add((s, DCTERMS.extent, Literal(md._iso_duration(row["duration_ms"]), datatype=XSD.duration)))
path = str(row.get("path") or "")
Expand Down
Loading
Loading
0