diff --git a/README.md b/README.md index 411a811b..8f82b2c8 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ [Documentation](https://digitalbazaar.github.io/pyld/) | [Installation](#installation) | [Usage](#usage) | [Advanced -Topics](#advanced-topics) | [Contributing](./CONTRIBUTING.md) | +Topics](#advanced-topics) | [What's new in version 4](#whats-new-in-version-4) | +[Contributing](./CONTRIBUTING.md) | [Changelog](./CHANGELOG.md) ## Introduction @@ -45,6 +46,27 @@ memory footprint in order to operate. * [Requests](http://docs.python-requests.org/) (optional) * [aiohttp](https://aiohttp.readthedocs.io/) (optional) +## What's new in version 4 + +PyLD 4 moves RDF handling to [RDFLib](https://rdflib.readthedocs.io/) and adds +RDF Dataset Canonicalization 1.0 support. + +The main developer-facing changes are: + +* `jsonld.to_rdf()` returns an `rdflib.Dataset` by default when no output + `format` is requested. +* `jsonld.to_rdf(doc, {'legacyMode': True})` returns the RDF.js-like dataset + `dict` used by PyLD versions lower than 4. +* `jsonld.from_rdf()` accepts `rdflib.Dataset`, N-Quads strings, and the legacy + dataset `dict`. +* `jsonld.normalize()` defaults to `RDFC10`; pass `{'algorithm': 'URDNA2015'}` + if older canonicalization output must remain stable. +* The internal `pyld.nquads` parser/serializer module has been removed. Use the + JSON-LD APIs with `format: 'application/n-quads'` or RDFLib directly. + +See the full [version 4 upgrade notes](https://digitalbazaar.github.io/pyld/whats-new-version-4/) +and the [`4.0.0` changelog entry](./CHANGELOG.md#400---unreleased). + ## Installation PyLD can be installed with a [pip](http://www.pip-installer.org/) @@ -121,10 +143,10 @@ flattened = jsonld.flatten(doc) framed = jsonld.frame(doc, frame) # document transformed into a particular tree structure per the given frame -# normalize a document using the RDF Dataset Normalization Algorithm -# (URDNA2015), see: https://www.w3.org/TR/rdf-canon/ +# normalize a document using RDF Dataset Canonicalization 1.0 +# (RDFC10), see: https://www.w3.org/TR/rdf-canon/ normalized = jsonld.normalize( - doc, {'algorithm': 'URDNA2015', 'format': 'application/n-quads'}) + doc, {'algorithm': 'RDFC10', 'format': 'application/n-quads'}) # normalized is a string that is a canonical representation of the document # that can be used for hashing, comparison, etc. ``` diff --git a/docs/.pages b/docs/.pages index a77446c5..d30acc84 100644 --- a/docs/.pages +++ b/docs/.pages @@ -1,8 +1,11 @@ nav: - Quickstart: - index.md + - whats-new-version-4.md - installation.md - conformance.md + - Tutorials: + - tutorials - Reference: - reference - Project: diff --git a/docs/examples/context_resolver_sqlite_cache.py b/docs/examples/context_resolver_sqlite_cache.py new file mode 100644 index 00000000..609ee559 --- /dev/null +++ b/docs/examples/context_resolver_sqlite_cache.py @@ -0,0 +1,21 @@ +import json +from pathlib import Path + +from cachetools import LRUCache + +from pyld import ContextResolver, SqliteCacheRequestsDocumentLoader, jsonld + +loader = SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path("/tmp/pyld_example_context_cache.sqlite"), +) +resolver = ContextResolver(LRUCache(maxsize=1000), loader) + +doc = { + "@context": "https://schema.org/", + "name": "Example Person", +} +options = {"documentLoader": loader, "contextResolver": resolver} + +expanded = jsonld.expand(doc, options=options) + +print(json.dumps(expanded, indent=2)) diff --git a/docs/examples/document_loaders/caching_modes.py b/docs/examples/document_loaders/caching_modes.py new file mode 100644 index 00000000..85fb03e1 --- /dev/null +++ b/docs/examples/document_loaders/caching_modes.py @@ -0,0 +1,31 @@ +import json +import tempfile +from pathlib import Path +from uuid import uuid4 + +from requests_cache import CachedSession + +from pyld import ( + RequestsDocumentLoader, + SqliteCacheRequestsDocumentLoader, + jsonld, +) + +doc = { + "@context": {"name": "https://schema.org/name"}, + "name": "Earth", +} + +no_cache_loader = RequestsDocumentLoader() + +memory_cache_loader = RequestsDocumentLoader( + session=CachedSession(backend="memory", cache_control=True), +) + +persistent_cache_loader = SqliteCacheRequestsDocumentLoader( + sqlite_file_path=Path(tempfile.gettempdir()) / f"pyld-{uuid4()}.sqlite", +) + +result = jsonld.expand(doc, options={"documentLoader": persistent_cache_loader}) + +print(json.dumps(result, indent=2)) diff --git a/docs/examples/from_rdf_dataset.py b/docs/examples/from_rdf_dataset.py new file mode 100644 index 00000000..58e8c6e7 --- /dev/null +++ b/docs/examples/from_rdf_dataset.py @@ -0,0 +1,15 @@ +import json + +from rdflib import Dataset + +from pyld import jsonld + +nquads = ( + ' ' + ' "Earth" .\n' +) +dataset = Dataset().parse(data=nquads, format="nquads") + +doc = jsonld.from_rdf(dataset) + +print(json.dumps(doc, indent=2)) diff --git a/docs/examples/inject_context.py b/docs/examples/inject_context.py new file mode 100644 index 00000000..cc3478ff --- /dev/null +++ b/docs/examples/inject_context.py @@ -0,0 +1,16 @@ +import json + +from pyld import jsonld + +doc = { + "name": "Ada Lovelace", + "homepage": "https://example.com/ada", +} +context = { + "name": "http://schema.org/name", + "homepage": {"@id": "http://schema.org/url", "@type": "@id"}, +} + +expanded = jsonld.expand(doc, {"expandContext": context}) + +print(json.dumps(expanded, indent=2)) diff --git a/docs/examples/inject_context_compact.py b/docs/examples/inject_context_compact.py new file mode 100644 index 00000000..06cbc579 --- /dev/null +++ b/docs/examples/inject_context_compact.py @@ -0,0 +1,24 @@ +import json + +from pyld import jsonld + +doc = { + "name": "Ada Lovelace", + "homepage": "https://example.com/ada", +} +source_context = { + "name": "http://schema.org/name", + "homepage": {"@id": "http://schema.org/url", "@type": "@id"}, +} +output_context = { + "name": "http://schema.org/name", + "url": {"@id": "http://schema.org/url", "@type": "@id"}, +} + +compacted = jsonld.compact( + doc, + output_context, + {"expandContext": source_context}, +) + +print(json.dumps(compacted, indent=2)) diff --git a/docs/examples/legacy_helpers.py b/docs/examples/legacy_helpers.py new file mode 100644 index 00000000..95375563 --- /dev/null +++ b/docs/examples/legacy_helpers.py @@ -0,0 +1,20 @@ +import json + +from pyld.util import from_legacy_dataset, to_legacy_dataset + +legacy_dataset = { + 'http://example.org': [ + { + 'subject': {'type': 'blank node', 'value': '_:s1'}, + 'predicate': {'type': 'IRI', 'value': 'p1'}, + 'object': {'type': 'literal', 'value': 'o1'}, + } + ] +} + + +rdflib_dataset = from_legacy_dataset(legacy_dataset) +print(rdflib_dataset.serialize(format="nquads")) + +legacy_dataset = to_legacy_dataset(rdflib_dataset) +print(json.dumps(legacy_dataset, indent=2)) diff --git a/docs/examples/normalize.py b/docs/examples/normalize.py index a6aa6d95..dcb600db 100644 --- a/docs/examples/normalize.py +++ b/docs/examples/normalize.py @@ -9,6 +9,15 @@ }, } +# Normalize with RDFC10 +normalized = jsonld.normalize( + doc, + {"format": "application/n-quads"}, +) + +print(normalized) + +# Normalize with URDNA2015 normalized = jsonld.normalize( doc, {"algorithm": "URDNA2015", "format": "application/n-quads"}, diff --git a/docs/examples/normalize_output_map.py b/docs/examples/normalize_output_map.py new file mode 100644 index 00000000..0ebbeea6 --- /dev/null +++ b/docs/examples/normalize_output_map.py @@ -0,0 +1,19 @@ +import json + +from pyld import jsonld + +doc = { + "@type": "http://schema.org/Person", + "http://schema.org/name": "Manu Sporny", + "http://schema.org/url": {"@id": "http://manu.sporny.org/"}, + "http://schema.org/image": { + "@id": "http://manu.sporny.org/images/manu.png" + }, +} + +identifier_map = jsonld.normalize( + doc, + {"algorithm": "RDFC10", "outputMap": True}, +) + +print(json.dumps(identifier_map, indent=2)) diff --git a/docs/examples/open_jsonld_file.py b/docs/examples/open_jsonld_file.py new file mode 100644 index 00000000..e8c64eac --- /dev/null +++ b/docs/examples/open_jsonld_file.py @@ -0,0 +1,11 @@ +import json +from pathlib import Path + +from pyld import jsonld + +path = Path(__file__).resolve().parent / "data" / "person.jsonld" +doc = json.loads(path.read_text()) + +expanded = jsonld.expand(doc) + +print(json.dumps(expanded, indent=2)) diff --git a/docs/examples/open_jsonld_file_to_rdf.py b/docs/examples/open_jsonld_file_to_rdf.py new file mode 100644 index 00000000..8b92e478 --- /dev/null +++ b/docs/examples/open_jsonld_file_to_rdf.py @@ -0,0 +1,11 @@ +import json +from pathlib import Path + +from pyld import jsonld + +path = Path(__file__).resolve().parent / "data" / "person.jsonld" +doc = json.loads(path.read_text()) + +nquads = jsonld.to_rdf(doc, {"format": "application/n-quads"}) + +print(nquads) diff --git a/docs/examples/persistent_document_loader_cache.py b/docs/examples/persistent_document_loader_cache.py new file mode 100644 index 00000000..bdb3ab39 --- /dev/null +++ b/docs/examples/persistent_document_loader_cache.py @@ -0,0 +1,18 @@ +import json +import tempfile +from pathlib import Path +from uuid import uuid4 + +from pyld import SqliteCacheRequestsDocumentLoader, jsonld + +cache_path = Path(tempfile.gettempdir()) / f"pyld-{uuid4()}.sqlite" +loader = SqliteCacheRequestsDocumentLoader(sqlite_file_path=cache_path) + +doc = { + "@context": {"name": "https://schema.org/name"}, + "name": "Earth", +} + +expanded = jsonld.expand(doc, options={"documentLoader": loader}) + +print(json.dumps(expanded, indent=2)) diff --git a/docs/examples/rdflib_parse.py b/docs/examples/rdflib_parse.py new file mode 100644 index 00000000..e76182e9 --- /dev/null +++ b/docs/examples/rdflib_parse.py @@ -0,0 +1,16 @@ +import json + +from rdflib import Dataset + +from pyld import jsonld + +nquads = ( + ' ' + ' "Earth" .\n' +) +dataset = Dataset().parse(data=nquads, format="nquads") + +doc = jsonld.from_rdf(dataset) +compacted = jsonld.compact(doc, {"name": "http://schema.org/name"}) + +print(json.dumps(compacted, indent=2)) diff --git a/docs/examples/rdflib_processing.py b/docs/examples/rdflib_processing.py new file mode 100644 index 00000000..71eebfa0 --- /dev/null +++ b/docs/examples/rdflib_processing.py @@ -0,0 +1,28 @@ +import json + +from rdflib import Namespace, URIRef + +from pyld import jsonld + +SCHEMA = Namespace("http://schema.org/") +subject = URIRef("http://dbpedia.org/resource/Earth") + +doc = { + "@context": {"name": str(SCHEMA.name)}, + "@id": str(subject), + "name": "Earth", +} + +dataset = jsonld.to_rdf(doc) +dataset.add((subject, SCHEMA.url, URIRef("https://example.com/earth"))) + +jsonld_doc = jsonld.from_rdf(dataset) +compacted = jsonld.compact( + jsonld_doc, + { + "name": str(SCHEMA.name), + "homepage": {"@id": str(SCHEMA.url), "@type": "@id"}, + }, +) + +print(json.dumps(compacted, indent=2)) diff --git a/docs/examples/rdflib_query.py b/docs/examples/rdflib_query.py new file mode 100644 index 00000000..bef96a87 --- /dev/null +++ b/docs/examples/rdflib_query.py @@ -0,0 +1,19 @@ +import json + +from rdflib import Namespace, URIRef + +from pyld import jsonld + +SCHEMA = Namespace("http://schema.org/") +subject = URIRef("http://dbpedia.org/resource/Earth") + +doc = { + "@context": {"name": str(SCHEMA.name)}, + "@id": str(subject), + "name": "Earth", +} + +dataset = jsonld.to_rdf(doc) +names = [str(value) for value in dataset.objects(subject, SCHEMA.name)] + +print(json.dumps(names, indent=2)) diff --git a/docs/examples/to_rdf_dataset.py b/docs/examples/to_rdf_dataset.py new file mode 100644 index 00000000..9f495652 --- /dev/null +++ b/docs/examples/to_rdf_dataset.py @@ -0,0 +1,14 @@ +from pyld import jsonld + +doc = { + "@context": { + "name": "http://schema.org/name", + }, + "@id": "http://dbpedia.org/resource/Earth", + "name": "Earth", +} + +dataset = jsonld.to_rdf(doc) + +print(type(dataset).__name__) +print(dataset.serialize(format="nquads")) diff --git a/docs/examples/to_rdf_legacy.py b/docs/examples/to_rdf_legacy.py new file mode 100644 index 00000000..735f87be --- /dev/null +++ b/docs/examples/to_rdf_legacy.py @@ -0,0 +1,15 @@ +import json + +from pyld import jsonld + +doc = { + "@context": { + "name": "http://schema.org/name", + }, + "@id": "http://dbpedia.org/resource/Earth", + "name": "Earth", +} + +legacy_dataset = jsonld.to_rdf(doc, {"legacyMode": True}) + +print(json.dumps(legacy_dataset, indent=2)) diff --git a/docs/index.md b/docs/index.md index ba0d37e4..4f649062 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,6 +15,15 @@ applications add meaning to existing JSON documents with in-band or out-of-band contexts, while keeping the document shape practical for web APIs, JavaScript, and JSON document stores. +## :material-new-box: What's new in version 4 + +PyLD 4 uses [:material-library: RDFLib](https://rdflib.readthedocs.io/) +`rdflib.Dataset` as the native RDF dataset representation, adds `RDFC10` +canonicalization, and removes the internal `pyld.nquads` parser/serializer +module. + +[Read the version 4 upgrade notes :octicons-arrow-right-24:](whats-new-version-4.md) + ## :material-lightning-bolt: Quick Examples === "jsonld.compact()" diff --git a/docs/reference/context-resolver.md b/docs/reference/context-resolver.md index c20a85bb..f45700a3 100644 --- a/docs/reference/context-resolver.md +++ b/docs/reference/context-resolver.md @@ -16,6 +16,12 @@ documents, so pass the same loader with `documentLoader`. {{ example('context_resolver.py', output_syntax='json') }} +Use the built-in `SqliteCacheRequestsDocumentLoader` when you want the resolver +to combine PyLD's in-memory resolved-context cache with persistent HTTP caching +for fetched remote contexts. + +{{ example('context_resolver_sqlite_cache.py', output_syntax='json') }} + The `shared_cache` object must behave like a mutable mapping. PyLD uses `cachetools.LRUCache` by default, but applications can provide another mapping when they need a different eviction policy. diff --git a/docs/reference/from_rdf.md b/docs/reference/from_rdf.md index 52ad7aba..b241dc3b 100644 --- a/docs/reference/from_rdf.md +++ b/docs/reference/from_rdf.md @@ -12,6 +12,17 @@ show_bases: false heading_level: 3 -## Example +`jsonld.from_rdf()` accepts a +[:material-library: RDFLib](https://rdflib.readthedocs.io/) `rdflib.Dataset`, +an N-Quads string, or the RDF.js-like dataset `dict` returned by PyLD versions +lower than 4. Prefer `rdflib.Dataset` for new in-memory RDF code. + +## Examples + +### Parsing N-Quads as JSON-LD {{ example('from_rdf.py', 'json') }} + +### Transforming an `rdflib.Dataset` into JSON-LD + +{{ example('from_rdf_dataset.py', 'json') }} diff --git a/docs/reference/normalize.md b/docs/reference/normalize.md index 7b05cad5..d1a7d830 100644 --- a/docs/reference/normalize.md +++ b/docs/reference/normalize.md @@ -12,6 +12,15 @@ show_bases: false heading_level: 3 -## Example +`RDFC10` is the default algorithm in PyLD 4. Use `URDNA2015` or `URGNA2012` +explicitly when an integration requires the older canonicalization output. + +## Examples + +### Normalizing RDF as canonical N-Quads {{ example('normalize.py') }} + +### Outputting the Identifier Map of the canonicalization process (RDFC1.0) + +{{ example('normalize_output_map.py', 'json') }} diff --git a/docs/reference/to_rdf.md b/docs/reference/to_rdf.md index 7d13bfb3..8dddd160 100644 --- a/docs/reference/to_rdf.md +++ b/docs/reference/to_rdf.md @@ -12,10 +12,24 @@ show_bases: false heading_level: 3 -When `format` is not set, `jsonld.to_rdf()` returns an -[`rdflib.Dataset`](https://rdflib.readthedocs.io/). Set `legacyMode` to `True` -to return the RDF.js-like dataset `dict` used by PyLD versions lower than 4. +When `format` is not set, `jsonld.to_rdf()` returns a +[:material-library: RDFLib](https://rdflib.readthedocs.io/) `rdflib.Dataset`. +Set `legacyMode` to `True` to return the RDF.js-like dataset `dict` used by +PyLD versions lower than 4. -## Example +`legacyMode` only affects the unformatted return value. If `format` is set, +`jsonld.to_rdf()` returns the requested serialization. + +## Examples + +### Serializing JSON-LD to N-Quads {{ example('to_rdf.py') }} + +### Transforming JSON-LD into an `rdflib.Dataset` + +{{ example('to_rdf_dataset.py') }} + +### Transforming JSON-LD into an legacy RDF.js-like dict + +{{ example('to_rdf_legacy.py', 'json') }} diff --git a/docs/tutorials/document-loader-caching.md b/docs/tutorials/document-loader-caching.md new file mode 100644 index 00000000..c6f3c5e1 --- /dev/null +++ b/docs/tutorials/document-loader-caching.md @@ -0,0 +1,54 @@ +# :material-cached: Caching remote contexts + +Remote JSON-LD contexts are normal documents loaded through a `DocumentLoader`. +Caching therefore belongs in two places: + +1. PyLD's `ContextResolver` caches resolved contexts during processing. +2. The `DocumentLoader` can cache HTTP responses before PyLD sees them. + +Use `RequestsDocumentLoader` when you want no HTTP cache. Use +`RequestsDocumentLoader` with a `requests_cache.CachedSession` for in-memory +HTTP caching. Use `SqliteCacheRequestsDocumentLoader` when the cache should +survive process restarts. + +For HTTP caching, install the necessary modules first by running: + +```bash +pip install "PyLD[requests-cache]" +``` + +## Choose a Cache Mode + +| Mode | Loader | Reused after process restart? | +| --- | --- | --- | +| No HTTP cache | `RequestsDocumentLoader()` | No | +| In-memory HTTP cache | `RequestsDocumentLoader(session=CachedSession(backend="memory"))` | No | +| Persistent HTTP cache | `SqliteCacheRequestsDocumentLoader()` | Yes | + +`SqliteCacheRequestsDocumentLoader` uses +[:simple-sqlite: SQLite](https://www.sqlite.org/) through +[:simple-pypi: `requests-cache`](https://pypi.org/project/requests-cache/). +When `sqlite_file_path` is omitted, it stores the cache in the platform user +cache directory. + +## Usage + +This example configures all three modes and passes the persistent loader through +`documentLoader`. The document uses an inline `@context` so the example is safe +to run without network access; use the same `options` shape when the `@context` +is a remote URL. + +{{ example('document_loaders/caching_modes.py', output_syntax='json') }} + +For application code, pass the selected loader in the operation `options`: + +{{ example('persistent_document_loader_cache.py', output_syntax='json') }} + +HTTP cache headers such as `Cache-Control`, `Expires`, and validators are +handled by the cached session. PyLD still applies its normal JSON-LD processing +rules after the remote document is loaded. + +??? note "Related reference" + See [`RequestsDocumentLoader`](../reference/document-loaders/requests.md), + [`SqliteCacheRequestsDocumentLoader`](../reference/document-loaders/sqlite-cache-requests.md), + and [`ContextResolver`](../reference/context-resolver.md). diff --git a/docs/tutorials/index.md b/docs/tutorials/index.md new file mode 100644 index 00000000..cd5b85e1 --- /dev/null +++ b/docs/tutorials/index.md @@ -0,0 +1,37 @@ +--- +hide: [toc] +--- + +# :material-school-outline: Tutorials + +
+ +- [:material-library:{ .lg .middle } RDFLib and `jsonld`](rdflib-processing.md) + + --- + + Convert JSON-LD to an `rdflib.Dataset`, process it with RDFLib, and convert + it back to JSON-LD. + +- [:material-code-json:{ .lg .middle } Inject a JSON-LD `@context`](inject-context.md) + + --- + + Apply a context to plain JSON when the source cannot provide an in-band + `@context` or HTTP `Link` header. + +- [:material-file-code-outline:{ .lg .middle } Open a JSON-LD file](open-jsonld-file.md) + + --- + + Load a local `.jsonld` file, process it with PyLD, and choose the right + dependencies for local or remote contexts. + +- [:material-cached:{ .lg .middle } Cache remote contexts with `DocumentLoader`](document-loader-caching.md) + + --- + + Combine PyLD's context cache with no-cache, in-memory, and persistent + document-loader HTTP caching. + +
diff --git a/docs/tutorials/inject-context.md b/docs/tutorials/inject-context.md new file mode 100644 index 00000000..ffd43430 --- /dev/null +++ b/docs/tutorials/inject-context.md @@ -0,0 +1,26 @@ +# :material-code-json: Inject a JSON-LD `@context` + +JSON-LD property names only become RDF terms when a context maps them to IRIs. +If the input JSON has no `@context`, and the source cannot provide one through +an HTTP `Link` header, pass the context with `expandContext`. + +Use this for JSON already loaded from a file, queue, database, or API client: + +1. Load or receive the plain JSON document. +2. Define the context your application wants to apply. +3. Pass that context in the operation `options` as `expandContext`. +4. Run `jsonld.expand()` or another API that expands internally. + +PyLD processes `expandContext` before expansion. If the document also has its +own `@context`, that document context is processed later and can refine or +override term definitions for the document body. + +{{ example('inject_context.py', 'json') }} + +The same option works with APIs that expand internally, including +`jsonld.compact()`, `jsonld.flatten()`, and `jsonld.frame()`: + +{{ example('inject_context_compact.py', 'json') }} + +Use an in-document `@context` when you control the JSON. Use `expandContext` +when the context is application policy or transport metadata. diff --git a/docs/tutorials/open-jsonld-file.md b/docs/tutorials/open-jsonld-file.md new file mode 100644 index 00000000..99e7cf9c --- /dev/null +++ b/docs/tutorials/open-jsonld-file.md @@ -0,0 +1,36 @@ +# :material-file-code-outline: Open a JSON-LD file + +For a local `.jsonld` file that already contains its `@context`, use Python's +standard `json` module first. PyLD processes Python objects, so the usual order +is: + +```bash +pip install PyLD +``` + +1. Read the file with `pathlib.Path`. +2. Parse it with `json.loads()` or `json.load()`. +3. Pass the parsed object to `jsonld.expand()`, `jsonld.compact()`, + `jsonld.to_rdf()`, or another PyLD API. + +{{ example('open_jsonld_file.py', 'json') }} + +After parsing the file, use any PyLD operation. For example, convert the local +JSON-LD file to N-Quads: + +{{ example('open_jsonld_file_to_rdf.py') }} + +That is enough for local files with inline contexts. Choose a document loader +only when PyLD must dereference a URL during processing. + +If the document uses remote contexts, install and pass +`RequestsDocumentLoader`: + +```bash +pip install "PyLD[requests]" +``` + +If PyLD should load the JSON-LD document itself from a `file:` URL, use +[`FileDocumentLoader`](../reference/document-loaders/file.md) and pass it with +`documentLoader` in the operation `options`. Direct `json.load()` is simpler +when you already know the file path and only need to process that one file. diff --git a/docs/tutorials/rdflib-processing.md b/docs/tutorials/rdflib-processing.md new file mode 100644 index 00000000..a3a35b65 --- /dev/null +++ b/docs/tutorials/rdflib-processing.md @@ -0,0 +1,42 @@ +# :material-library: RDFLib and `jsonld` + +Use this pattern when JSON-LD is your interchange format, but RDF tooling is the +best place to query, update, serialize, or store the graph. + +Install PyLD: + +```bash +pip install PyLD +``` + +PyLD 4 installs and uses [:material-library: RDFLib](https://rdflib.readthedocs.io/) +for RDF datasets. The processing order is: + +1. Start with JSON-LD that has a usable `@context`. +2. Convert it with `jsonld.to_rdf()`. +3. Process the returned `rdflib.Dataset` with RDFLib. +4. Convert it back with `jsonld.from_rdf()` if the next layer expects JSON-LD. +5. Compact the result with `jsonld.compact()` if developers or APIs should see + terms such as `name` instead of full IRIs. + +{{ example('rdflib_processing.py', 'json') }} + +For simple lookups, use RDFLib's graph methods on the dataset returned by +`jsonld.to_rdf()`: + +{{ example('rdflib_query.py', 'json') }} + +For RDF that starts outside PyLD, parse it with RDFLib first, then pass the +`rdflib.Dataset` to `jsonld.from_rdf()`: + +{{ example('rdflib_parse.py', 'json') }} + +Use `format` only when you need a serialized RDF string instead of an in-memory +`rdflib.Dataset`: + +```python +nquads = jsonld.to_rdf(doc, {"format": "application/n-quads"}) +``` + +Use `jsonld.from_rdf(dataset)` directly when you already have an +`rdflib.Dataset` from another RDFLib parser, store, or query result. diff --git a/docs/whats-new-version-4.md b/docs/whats-new-version-4.md new file mode 100644 index 00000000..d2ff3050 --- /dev/null +++ b/docs/whats-new-version-4.md @@ -0,0 +1,98 @@ +# :material-new-box: What's new in version 4 + +PyLD 4 moves the RDF datastructure to [:material-library: RDFLib](https://rdflib.readthedocs.io/) +and adds RDF Dataset Canonicalization 1.0 support. The JSON-LD document APIs +remain the same, but RDF-facing code should review return types, canonicalization +defaults, and N-Quads behavior before upgrading. + +## RDFLib datasets are now the native RDF model + +`jsonld.to_rdf()` returns an `rdflib.Dataset` by default when `format` is not +set. In PyLD 3.x and earlier, it returned a RDF.js-like nested `dict`. + +{{ example('to_rdf_dataset.py') }} + +Request N-Quads when you need a serialized string: + +{{ example('to_rdf.py') }} + +Use `legacyMode` when existing code still expects the PyLD 3.x dataset `dict`: + +{{ example('to_rdf_legacy.py', 'json') }} + +`jsonld.from_rdf()` accepts an `rdflib.Dataset`, an N-Quads string, or the +legacy dataset `dict`. New code should prefer `rdflib.Dataset` for in-memory RDF +work and `application/n-quads` for process or storage boundaries. + +## `RDFC10` is available and is the normalization default + +`jsonld.normalize()` now defaults to `RDFC10`, the RDF Dataset Canonicalization +1.0 algorithm. `URDNA2015` and `URGNA2012` remain available by setting +`algorithm` explicitly. + +{{ example('normalize.py') }} + +For RDFC 1.0 test vectors and integrations that need the canonical blank node +identifier map, pass `outputMap`: + +{{ example('normalize_output_map.py', 'json') }} + +`RDFC10` also accepts `hashAlgorithm` for test suites and specialized +integrations. Most applications should keep the default SHA-256 behavior. + +## N-Quads parsing and serialization delegates to RDFLib + +PyLD 4 removes the internal `pyld.nquads` parser and serializer module. Public +JSON-LD APIs still accept and produce N-Quads through `format: "application/n-quads"`, +but imports from `pyld.nquads` need to be removed. + +If you previously used `pyld.nquads` directly, replace it with one of these +paths: + +- Use `jsonld.from_rdf(nquads, {"format": "application/n-quads"})` to convert + N-Quads to JSON-LD. +- Use `jsonld.to_rdf(doc, {"format": "application/n-quads"})` to serialize + JSON-LD as N-Quads. +- Use `jsonld.parse_nquads(doc, {"legacyMode": True})` to convert nquads + to a RDF.js-like nested `dict` from PyLD 3.x and earlier. + Omit `legacyMode` to return an `rdflib.Dataset`. + This method preserves blank node identifiers from the input document. +- Use `rdflib.Dataset().parse(data=nquads, format="nquads")` or + `rdflib.plugins.parsers.nquads.NQuadsParser()` for direct RDFLib parsing. + Note that, opposed to `jsonld.parse_nquads`, this does NOT preserve blank node + identifiers by default. + +## Compatibility helpers + +PyLD 4 includes conversion helpers for applications that need to bridge between +the old in-memory RDF.js-like nested `dict` and RDFLib: + +{{ example('legacy_helpers.py') }} + +Treat these helpers as migration aids. Prefer RDFLib terms and datasets in new +code so RDF processing is compatible with the rest of the Python RDF ecosystem. + +## Behavior fixes to expect + +The RDFLib migration also fixes several RDF conversion edge cases: + +- RDF literal lexical forms are preserved more carefully through RDFLib + conversion, including canonical double output, large numeric values, and + compound literals. +- Invalid IRI and language values are skipped during `jsonld.to_rdf()` instead + of producing invalid triples or crashing. +- Query and fragment reconstruction in `iri_resolver.unresolve()` is corrected. +- More W3C URDNA2015, URDNA2012, RDFC10, and JSON-LD `toRdf` tests run through + the default test runner. + +## Upgrade checklist + +- Add `rdflib` to application constraints if dependencies are pinned outside + PyLD's package metadata. +- Audit `jsonld.to_rdf()` call sites that do not pass `format`. Update them to + handle `rdflib.Dataset` or temporarily pass `legacyMode`. +- Remove imports of `pyld.nquads`. +- Check normalization call sites that relied on the old default algorithm. Pass + `{"algorithm": "URDNA2015"}` explicitly if that output must remain stable. +- Compare N-Quads as RDF data or sorted lines in tests unless the test requires + exact serializer ordering. diff --git a/lib/pyld/options.py b/lib/pyld/options.py index 305bed35..6b87605c 100644 --- a/lib/pyld/options.py +++ b/lib/pyld/options.py @@ -149,12 +149,18 @@ class NormalizeOptions(ProcessingOptions, total=False): algorithm: Literal['URDNA2015', 'URGNA2012', 'RDFC10'] """The algorithm to use (default: `RDFC10`).""" + hashAlgorithm: str + """The hashing algorithm for `RDFC10` (default: `SHA256`).""" + inputFormat: Literal['application/n-quads'] """The format if input is not JSON-LD: `application/n-quads` for N-Quads.""" format: Literal['application/n-quads'] """The format if output is a string: `application/n-quads` for N-Quads.""" + outputMap: bool + """`True` to return a blank node identifier map instead of the normalized dataset (default: `False`).""" + class ToRdfOptions(ProcessingOptions, total=False): documentLoader: DocumentLoader | DocumentLoaderCallable