Use IA² with HTML

import { JSDOM } from "jsdom";
import { CARRIER_RULES } from "./carrier-profile.mjs";

const XSD_STRING = "http://www.w3.org/2001/XMLSchema#string";
const RDF_DIR_LANG_STRING = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString";

// Fetch and parse the document with its retrieval URL as the DOM base URL.
const response = await fetch(process.argv[2]);
if (!response.ok) {
  throw new Error(`HTTP ${response.status}`);
}
const html = await response.text();
const document = new JSDOM(html, { url: response.url }).window.document;

// HTML/RDF resolves semantic references against the one canonical source IRI.
const canonical = [...document.head.querySelectorAll("link[rel][href]")]
  .filter((link) => link.relList.contains("canonical"));
if (canonical.length !== 1) {
  throw new Error("Expected one canonical link");
}
const sourceIri = new URL(canonical[0].getAttribute("href"), response.url).href;

// A normal attribute selector finds every locally complete statement carrier.
const rows = [...document.querySelectorAll("[rdf-predicate]")]
  .filter((element) => !element.parentElement?.closest("template"))
  .map((element) => {
    const subject = new URL(element.getAttribute("rdf-subject"), sourceIri).href;
    const predicate = new URL(element.getAttribute("rdf-predicate"), sourceIri).href;

    // The generated profile identifies any native attribute that carries the value.
    const carriers = CARRIER_RULES.filter((rule) =>
      rule.element === element.localName && element.hasAttribute(rule.attribute));
    if (carriers.length > 1) {
      throw new Error(`Conflicting value carriers on <${element.localName}>`);
    }
    const carrier = carriers[0];

    // IRI carrier values are references; resolve them against the semantic source.
    if (carrier?.kind === "iri") {
      const object = new URL(element.getAttribute(carrier.attribute), sourceIri).href;
      return [subject, predicate, "I", object, "", "", ""];
    }

    // Literal carriers are exact strings. Other elements use normalized visible text.
    const value = carrier?.kind === "literal"
      ? element.getAttribute(carrier.attribute)
      : element.textContent.replace(/[\t\n\f\r ]+/g, " ").trim();

    // lang and dir apply only when they are directly present on this carrier.
    const language = element.getAttribute("lang") ?? "";
    const rawDirection = element.getAttribute("dir")?.toLowerCase();
    const direction = ["ltr", "rtl"].includes(rawDirection)
      ? rawDirection
      : "";

    // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    let datatype = direction ? RDF_DIR_LANG_STRING : XSD_STRING;
    if (element.hasAttribute("rdf-datatype")) {
      datatype = new URL(element.getAttribute("rdf-datatype"), sourceIri).href;
    }
    return [subject, predicate, "L", value, datatype, language, direction];
  })
  .sort((left, right) => left[1].localeCompare(right[1]));

process.stdout.write(`${rows.map((row) => row.join("\t")).join("\n")}\n`);

from __future__ import annotations

import re
import sys
from urllib.parse import urljoin

import html5lib
import httpx
from carrier_profile import CARRIER_RULES

XSD_STRING = "http://www.w3.org/2001/XMLSchema#string"
RDF_DIR_LANG_STRING = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"


def local_name(element) -> str:
    return element.tag.rsplit("}", 1)[-1]


# Fetch and parse with the HTML5 tree builder rather than an XML parser.
response = httpx.get(sys.argv[1], follow_redirects=True, timeout=10)
response.raise_for_status()
document = html5lib.parse(
    response.content,
    treebuilder="lxml",
    namespaceHTMLElements=True,
    transport_encoding=response.encoding,
)

# HTML/RDF resolves semantic references against the one canonical source IRI.
canonical = [
    element
    for element in document.xpath("//*[local-name()='head']/*[local-name()='link'][@rel][@href]")
    if "canonical" in element.get("rel", "").lower().split()
]
if len(canonical) != 1:
    raise RuntimeError("Expected one canonical link")
source_iri = urljoin(str(response.url), canonical[0].get("href"))

# XPath provides the same attribute-name discovery query as [rdf-predicate].
rows: list[list[str]] = []
for element in document.xpath("//*[@rdf-predicate]"):
    if any(local_name(parent) == "template" for parent in element.iterancestors()):
        continue
    subject = urljoin(source_iri, element.get("rdf-subject"))
    predicate = urljoin(source_iri, element.get("rdf-predicate"))

    # The generated profile identifies any native attribute that carries the value.
    carriers = [
        rule
        for rule in CARRIER_RULES
        if rule[0] == local_name(element) and element.get(rule[1]) is not None
    ]
    if len(carriers) > 1:
        raise RuntimeError(f"Conflicting value carriers on <{local_name(element)}>")
    carrier = carriers[0] if carriers else None

    # IRI carrier values are references; resolve them against the semantic source.
    if carrier and carrier[2] == "iri":
        object_iri = urljoin(source_iri, element.get(carrier[1]))
        rows.append([subject, predicate, "I", object_iri, "", "", ""])
        continue

    # Literal carriers are exact strings. Other elements use normalized visible text.
    if carrier and carrier[2] == "literal":
        value = element.get(carrier[1])
    else:
        value = re.sub(r"[\t\n\f\r ]+", " ", "".join(element.itertext())).strip()

    # lang and dir apply only when they are directly present on this carrier.
    language = element.get("lang", "")
    direction = element.get("dir", "").lower()
    direction = direction if direction in {"ltr", "rtl"} else ""

    # An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    datatype = RDF_DIR_LANG_STRING if direction else XSD_STRING
    if element.get("rdf-datatype") is not None:
        datatype = urljoin(source_iri, element.get("rdf-datatype"))
    rows.append([subject, predicate, "L", value, datatype, language, direction])

for row in sorted(rows, key=lambda item: item[1]):
    print("\t".join(row))

package dev.ia2.example;

import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

public final class Extract {
  private static final String XSD_STRING = "http://www.w3.org/2001/XMLSchema#string";
  private static final String RDF_DIR_LANG_STRING =
      "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString";

  private static String resolve(String base, String reference) {
    return URI.create(base).resolve(reference).toString();
  }

  public static void main(String[] args) throws Exception {
    // Fetch and parse the document as HTML.
    var client = HttpClient.newBuilder().followRedirects(HttpClient.Redirect.NORMAL).build();
    var request = HttpRequest.newBuilder(URI.create(args[0])).GET().build();
    var response = client.send(request, HttpResponse.BodyHandlers.ofByteArray());
    if (response.statusCode() < 200 || response.statusCode() >= 300) {
      throw new IllegalStateException("HTTP " + response.statusCode());
    }
    var html = new String(response.body(), StandardCharsets.UTF_8);
    Document document = Jsoup.parse(html, response.uri().toString());

    // HTML/RDF resolves semantic references against one canonical source IRI.
    List<Element> canonical = document.select("head link[rel][href]").stream()
        .filter(link -> List.of(link.attr("rel").split("\\s+")).stream()
            .anyMatch("canonical"::equalsIgnoreCase))
        .toList();
    if (canonical.size() != 1) {
      throw new IllegalStateException("Expected one canonical link");
    }
    String sourceIri = resolve(response.uri().toString(), canonical.get(0).attr("href"));

    // A normal attribute selector finds every locally complete carrier.
    List<List<String>> rows = new ArrayList<>();
    for (Element element : document.select("[rdf-predicate]")) {
      boolean insideTemplate = element.parents().stream()
          .anyMatch(parent -> parent.normalName().equals("template"));
      if (insideTemplate) {
        continue;
      }
      String subject = resolve(sourceIri, element.attr("rdf-subject"));
      String predicate = resolve(sourceIri, element.attr("rdf-predicate"));

      // The generated profile identifies any native attribute that carries the value.
      List<CarrierRule> carriers = CarrierProfile.RULES.stream()
          .filter(rule -> rule.element().equals(element.normalName()))
          .filter(rule -> element.hasAttr(rule.attribute()))
          .toList();
      if (carriers.size() > 1) {
        throw new IllegalStateException("Conflicting value carriers on <"
            + element.normalName() + ">");
      }
      CarrierRule carrier = carriers.isEmpty() ? null : carriers.get(0);

      // IRI carrier values are references; resolve them against the semantic source.
      if (carrier != null && carrier.kind().equals("iri")) {
        String objectIri = resolve(sourceIri, element.attr(carrier.attribute()));
        rows.add(List.of(subject, predicate, "I", objectIri, "", "", ""));
        continue;
      }

      // Literal carriers are exact strings. Other elements use normalized visible text.
      String value = carrier != null && carrier.kind().equals("literal")
          ? element.attr(carrier.attribute())
          : element.text();

      // lang and dir apply only when directly present on this carrier.
      String language = element.attr("lang");
      String direction = element.attr("dir").toLowerCase();
      if (!direction.equals("ltr") && !direction.equals("rtl")) {
        direction = "";
      }

      // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
      String datatype = direction.isEmpty() ? XSD_STRING : RDF_DIR_LANG_STRING;
      if (element.hasAttr("rdf-datatype")) {
        datatype = resolve(sourceIri, element.attr("rdf-datatype"));
      }
      rows.add(List.of(subject, predicate, "L", value, datatype, language, direction));
    }
    rows.sort(Comparator.comparing(row -> row.get(1)));
    rows.forEach(row -> System.out.println(String.join("\t", row)));
  }
}

import io.ktor.client.HttpClient
import io.ktor.client.engine.cio.CIO
import io.ktor.client.request.get
import io.ktor.client.statement.bodyAsText
import java.net.URI
import kotlinx.coroutines.runBlocking
import org.jsoup.Jsoup

private const val XSD_STRING = "http://www.w3.org/2001/XMLSchema#string"
private const val RDF_DIR_LANG_STRING = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"
private fun resolve(base: String, reference: String) =
    URI.create(base).resolve(reference).toString()

fun main(args: Array<String>) = runBlocking {
    HttpClient(CIO) { followRedirects = true }.use { client ->
        // Fetch and parse the document as HTML.
        val response = client.get(args[0])
        val retrievalIri = response.call.request.url.toString()
        val document = Jsoup.parse(response.bodyAsText(), retrievalIri)

        // Resolve semantic references against the one canonical source IRI.
        val canonical = document.select("head link[rel][href]").filter {
            it.attr("rel").split(Regex("\\s+")).any { token ->
                token.equals("canonical", ignoreCase = true)
            }
        }
        require(canonical.size == 1) { "Expected one canonical link" }
        val sourceIri = resolve(retrievalIri, canonical.first()!!.attr("href"))

        // A normal attribute selector finds every locally complete carrier.
        val rows = document.select("[rdf-predicate]").asSequence()
            .filterNot { it.parents().any { parent -> parent.normalName() == "template" } }
            .map { element ->
                val subject = resolve(sourceIri, element.attr("rdf-subject"))
                val predicate = resolve(sourceIri, element.attr("rdf-predicate"))

                // The generated profile identifies any native attribute carrying the value.
                val valueCarriers = carrierRules.filter { rule ->
                    rule.element == element.normalName() && element.hasAttr(rule.attribute)
                }
                require(valueCarriers.size <= 1) {
                    "Conflicting value carriers on <${element.normalName()}>"
                }
                val carrier = valueCarriers.singleOrNull()

                // IRI carrier values are references; resolve them against the semantic source.
                if (carrier?.kind == "iri") {
                    val objectIri = resolve(sourceIri, element.attr(carrier.attribute))
                    listOf(subject, predicate, "I", objectIri, "", "", "")
                } else {
                    // Literal carriers are exact strings. Other elements use normalized text.
                    val value = carrier?.takeIf { it.kind == "literal" }
                        ?.let { element.attr(it.attribute) } ?: element.text()

                    // lang and dir apply only when directly present on this carrier.
                    val language = element.attr("lang")
                    val direction = element.attr("dir").lowercase()
                        .takeIf { it == "ltr" || it == "rtl" } ?: ""

                    // An explicit datatype wins; direction otherwise selects dirLangString.
                    val datatype = when {
                        element.hasAttr("rdf-datatype") ->
                            resolve(sourceIri, element.attr("rdf-datatype"))
                        direction.isNotEmpty() -> RDF_DIR_LANG_STRING
                        else -> XSD_STRING
                    }
                    listOf(subject, predicate, "L", value, datatype, language, direction)
                }
            }.sortedBy { it[1] }
        rows.forEach { println(it.joinToString("\t")) }
    }
}

//> using scala "3.7.1"
//> using dep "com.softwaremill.sttp.client3::core:3.11.0"
//> using dep "org.jsoup:jsoup:1.22.2"

import java.net.URI
import org.jsoup.Jsoup
import scala.jdk.CollectionConverters.*
import sttp.client3.*

@main def extract(url: String): Unit =
  // Fetch and parse the document as HTML.
  val backend = HttpURLConnectionBackend()
  val response = basicRequest.get(uri"$url").send(backend)
  val body = response.body.fold(error => throw RuntimeException(error), identity)
  val retrievalIri = url
  val document = Jsoup.parse(body, retrievalIri)

  // Resolve semantic references against the one canonical source IRI.
  val canonical = document.select("head link[rel][href]").asScala.filter { link =>
    link.attr("rel").split("\\s+").exists(_.equalsIgnoreCase("canonical"))
  }
  require(canonical.size == 1, "Expected one canonical link")
  def resolve(base: String, reference: String) = URI.create(base).resolve(reference).toString
  val sourceIri = resolve(retrievalIri, canonical.head.attr("href"))
  val xsdString = "http://www.w3.org/2001/XMLSchema#string"
  val rdfDirLangString = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

  // A normal attribute selector finds every locally complete carrier.
  val rows = document.select("[rdf-predicate]").asScala
    .filterNot(_.parents().asScala.exists(_.normalName() == "template"))
    .map { element =>
      val subject = resolve(sourceIri, element.attr("rdf-subject"))
      val predicate = resolve(sourceIri, element.attr("rdf-predicate"))

      // The generated profile identifies any native attribute that carries the value.
      val carriers = carrierRules.filter { rule =>
        rule.element == element.normalName() && element.hasAttr(rule.attribute)
      }
      require(carriers.size <= 1, s"Conflicting value carriers on <${element.normalName()}>")
      val carrier = carriers.headOption

      // IRI carrier values are references; resolve them against the semantic source.
      if carrier.exists(_.kind == "iri") then
        val objectIri = resolve(sourceIri, element.attr(carrier.get.attribute))
        Seq(subject, predicate, "I", objectIri, "", "", "")
      else
        // Literal carriers are exact strings. Other elements use normalized visible text.
        val value = carrier.filter(_.kind == "literal")
          .map(rule => element.attr(rule.attribute))
          .getOrElse(element.text())

        // lang and dir apply only when directly present on this carrier.
        val language = element.attr("lang")
        val rawDirection = element.attr("dir").toLowerCase
        val direction = if rawDirection == "ltr" || rawDirection == "rtl" then rawDirection else ""

        // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
        val datatype =
          if element.hasAttr("rdf-datatype") then
            resolve(sourceIri, element.attr("rdf-datatype"))
          else if direction.nonEmpty then
            rdfDirLangString
          else
            xsdString
        Seq(subject, predicate, "L", value, datatype, language, direction)
    }.sortBy(_(1))
  rows.foreach(row => println(row.mkString("\t")))
  backend.close()

using System.Text.RegularExpressions;
using AngleSharp.Dom;
using AngleSharp.Html.Parser;

const string XsdString = "http://www.w3.org/2001/XMLSchema#string";
const string RdfDirLangString = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString";

static string Resolve(string baseIri, string reference) =>
    new Uri(new Uri(baseIri), reference).AbsoluteUri;
static bool InsideTemplate(IElement element)
{
    for (var parent = element.ParentElement; parent is not null; parent = parent.ParentElement)
    {
        if (parent.LocalName == "template")
        {
            return true;
        }
    }
    return false;
}

// Fetch and parse the document as HTML.
using var client = new HttpClient(new HttpClientHandler { AllowAutoRedirect = true });
using var response = await client.GetAsync(args[0]);
response.EnsureSuccessStatusCode();
var retrievalIri = response.RequestMessage!.RequestUri!.AbsoluteUri;
var html = await response.Content.ReadAsStringAsync();
var document = await new HtmlParser().ParseDocumentAsync(html);

// Resolve semantic references against the one canonical source IRI.
var canonical = document.Head!.QuerySelectorAll("link[rel][href]")
    .Where(link => link.GetAttribute("rel")!
        .Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries)
        .Any(token => token.Equals("canonical", StringComparison.OrdinalIgnoreCase)))
    .ToList();
if (canonical.Count != 1)
{
    throw new InvalidOperationException("Expected one canonical link");
}
var sourceIri = Resolve(retrievalIri, canonical[0].GetAttribute("href")!);

// A normal attribute selector finds every locally complete carrier.
var rows = new List<string[]>();
var carriers = document.QuerySelectorAll("[rdf-predicate]")
    .Where(element => !InsideTemplate(element));
foreach (var element in carriers)
{
    var subject = Resolve(sourceIri, element.GetAttribute("rdf-subject")!);
    var predicate = Resolve(sourceIri, element.GetAttribute("rdf-predicate")!);

    // The generated profile identifies any native attribute that carries the value.
    var valueCarriers = CarrierProfile.Rules
        .Where(rule => rule.Element == element.LocalName && element.HasAttribute(rule.Attribute))
        .ToList();
    if (valueCarriers.Count > 1)
    {
        throw new InvalidOperationException(Source is added from the example file during the site build.quot;Conflicting value carriers on <{element.LocalName}>");
    }
    CarrierRule? carrier = valueCarriers.Count == 0 ? null : valueCarriers[0];

    // IRI carrier values are references; resolve them against the semantic source.
    if (carrier is { Kind: "iri" } iriCarrier)
    {
        var objectIri = Resolve(sourceIri, element.GetAttribute(iriCarrier.Attribute)!);
        rows.Add([subject, predicate, "I", objectIri, "", "", ""]);
        continue;
    }

    // Literal carriers are exact strings. Other elements use normalized visible text.
    var value = carrier is { Kind: "literal" } literalCarrier
        ? element.GetAttribute(literalCarrier.Attribute)!
        : Regex.Replace(element.TextContent, "[\\t\\n\\f\\r ]+", " ").Trim();

    // lang and dir apply only when directly present on this carrier.
    var language = element.GetAttribute("lang") ?? "";
    var rawDirection = (element.GetAttribute("dir") ?? "").ToLowerInvariant();
    var direction = rawDirection is "ltr" or "rtl" ? rawDirection : "";

    // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    var datatype = direction.Length > 0 ? RdfDirLangString : XsdString;
    if (element.HasAttribute("rdf-datatype"))
    {
        datatype = Resolve(sourceIri, element.GetAttribute("rdf-datatype")!);
    }
    rows.Add([subject, predicate, "L", value, datatype, language, direction]);
}
foreach (var row in rows.OrderBy(row => row[1]))
{
    Console.WriteLine(string.Join('\t', row));
}

package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html/charset"
)

const xsdString = "http://www.w3.org/2001/XMLSchema#string"
const rdfDirLangString = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

func resolve(base, reference string) string {
	b, err := url.Parse(base)
	if err != nil {
		panic(err)
	}
	r, err := url.Parse(reference)
	if err != nil {
		panic(err)
	}
	return b.ResolveReference(r).String()
}

func main() {
	// Fetch and parse the document as HTML.
	response, err := http.Get(os.Args[1])
	if err != nil {
		panic(err)
	}
	defer response.Body.Close()
	if response.StatusCode < 200 || response.StatusCode >= 300 {
		panic(response.Status)
	}
	reader, err := charset.NewReader(response.Body, response.Header.Get("Content-Type"))
	if err != nil {
		panic(err)
	}
	document, err := goquery.NewDocumentFromReader(reader)
	if err != nil {
		panic(err)
	}

	// Resolve semantic references against the one canonical source IRI.
	var canonical []string
	document.Find("head link[rel][href]").Each(func(_ int, selection *goquery.Selection) {
		rel, _ := selection.Attr("rel")
		for _, token := range strings.Fields(strings.ToLower(rel)) {
			if token == "canonical" {
				href, _ := selection.Attr("href")
				canonical = append(canonical, href)
			}
		}
	})
	if len(canonical) != 1 {
		panic("expected one canonical link")
	}
	sourceIRI := resolve(response.Request.URL.String(), canonical[0])

	// A normal attribute selector finds every locally complete carrier.
	rows := make([][]string, 0, 4)
	document.Find("[rdf-predicate]").Each(func(_ int, element *goquery.Selection) {
		if element.ParentsFiltered("template").Length() != 0 {
			return
		}
		subjectRef, _ := element.Attr("rdf-subject")
		predicateRef, _ := element.Attr("rdf-predicate")
		subject := resolve(sourceIRI, subjectRef)
		predicate := resolve(sourceIRI, predicateRef)

		// The generated profile identifies any native attribute carrying the value.
		var carriers []carrierRule
		for _, rule := range carrierRules {
			_, present := element.Attr(rule.attribute)
			if rule.element == goquery.NodeName(element) && present {
				carriers = append(carriers, rule)
			}
		}
		if len(carriers) > 1 {
			panic("conflicting value carriers on <" + goquery.NodeName(element) + ">")
		}
		var carrier *carrierRule
		if len(carriers) == 1 {
			carrier = &carriers[0]
		}

		// IRI carrier values are references; resolve them against the semantic source.
		if carrier != nil && carrier.kind == "iri" {
			value, _ := element.Attr(carrier.attribute)
			rows = append(rows, []string{subject, predicate, "I", resolve(sourceIRI, value), "", "", ""})
			return
		}

		// Literal carriers are exact strings. Other elements use normalized text.
		value := element.Text()
		if carrier != nil && carrier.kind == "literal" {
			value, _ = element.Attr(carrier.attribute)
		} else {
			value = strings.Join(strings.Fields(value), " ")
		}

		// lang and dir apply only when directly present on this carrier.
		language, _ := element.Attr("lang")
		direction, _ := element.Attr("dir")
		direction = strings.ToLower(direction)
		if direction != "ltr" && direction != "rtl" {
			direction = ""
		}

		// An explicit datatype wins; direction otherwise selects rdf:dirLangString.
		datatype := xsdString
		if declared, ok := element.Attr("rdf-datatype"); ok {
			datatype = resolve(sourceIRI, declared)
		} else if direction != "" {
			datatype = rdfDirLangString
		}
		rows = append(rows, []string{subject, predicate, "L", value, datatype, language, direction})
	})
	sort.Slice(rows, func(i, j int) bool { return rows[i][1] < rows[j][1] })
	for _, row := range rows {
		fmt.Println(strings.Join(row, "\t"))
	}
}

mod carrier_profile;

use carrier_profile::CARRIER_RULES;
use scraper::{ElementRef, Html, Selector};
use std::error::Error;
use url::Url;

const XSD_STRING: &str = "http://www.w3.org/2001/XMLSchema#string";
const RDF_DIR_LANG_STRING: &str = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString";

fn normalize(value: &str) -> String {
    value.split_whitespace().collect::<Vec<_>>().join(" ")
}

fn inside_template(element: &ElementRef<'_>) -> bool {
    element
        .ancestors()
        .skip(1)
        .filter_map(ElementRef::wrap)
        .any(|ancestor| ancestor.value().name() == "template")
}

fn main() -> Result<(), Box<dyn Error>> {
    // Fetch and parse the document as HTML.
    let response = reqwest::blocking::get(std::env::args().nth(1).ok_or("missing URL")?)?
        .error_for_status()?;
    let retrieval_iri = response.url().clone();
    let document = Html::parse_document(&response.text()?);

    // Resolve semantic references against the one canonical source IRI.
    let links = Selector::parse("head link[rel][href]").unwrap();
    let canonical: Vec<_> = document
        .select(&links)
        .filter(|element| {
            element
                .value()
                .attr("rel")
                .unwrap_or("")
                .split_ascii_whitespace()
                .any(|token| token.eq_ignore_ascii_case("canonical"))
        })
        .collect();
    if canonical.len() != 1 {
        return Err("expected one canonical link".into());
    }
    let source_iri = retrieval_iri.join(canonical[0].value().attr("href").unwrap())?;

    // A normal attribute selector finds every locally complete carrier.
    let statements = Selector::parse("[rdf-predicate]").unwrap();
    let mut rows: Vec<Vec<String>> = Vec::new();
    for element in document
        .select(&statements)
        .filter(|element| !inside_template(element))
    {
        let subject = source_iri
            .join(element.value().attr("rdf-subject").unwrap())?
            .to_string();
        let predicate = source_iri
            .join(element.value().attr("rdf-predicate").unwrap())?
            .to_string();

        // The generated profile identifies any native attribute carrying the value.
        let carriers: Vec<_> = CARRIER_RULES
            .iter()
            .filter(|rule| {
                rule.element == element.value().name()
                    && element.value().attr(rule.attribute).is_some()
            })
            .collect();
        if carriers.len() > 1 {
            return Err(
                format!("conflicting value carriers on <{}>", element.value().name()).into(),
            );
        }
        let carrier = carriers.first().copied();

        // IRI carrier values are references; resolve them against the semantic source.
        if let Some(rule) = carrier.filter(|rule| rule.kind == "iri") {
            let object = source_iri
                .join(element.value().attr(rule.attribute).unwrap())?
                .to_string();
            rows.push(vec![
                subject,
                predicate,
                "I".into(),
                object,
                "".into(),
                "".into(),
                "".into(),
            ]);
            continue;
        }

        // Literal carriers are exact strings. Other elements use normalized visible text.
        let value = if let Some(rule) = carrier.filter(|rule| rule.kind == "literal") {
            element.value().attr(rule.attribute).unwrap().to_owned()
        } else {
            normalize(&element.text().collect::<String>())
        };

        // lang and dir apply only when directly present on this carrier.
        let language = element.value().attr("lang").unwrap_or("").to_owned();
        let direction = match element
            .value()
            .attr("dir")
            .unwrap_or("")
            .to_ascii_lowercase()
            .as_str()
        {
            "ltr" => "ltr",
            "rtl" => "rtl",
            _ => "",
        }
        .to_owned();

        // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
        let datatype = match element.value().attr("rdf-datatype") {
            Some(value) => source_iri.join(value)?.to_string(),
            None if !direction.is_empty() => RDF_DIR_LANG_STRING.into(),
            None => XSD_STRING.into(),
        };
        rows.push(vec![
            subject,
            predicate,
            "L".into(),
            value,
            datatype,
            language,
            direction,
        ]);
    }
    rows.sort_by(|left, right| left[1].cmp(&right[1]));
    for row in rows {
        println!("{}", row.join("\t"));
    }
    let _: Url = source_iri;
    Ok(())
}

<?php

require __DIR__ . '/vendor/autoload.php';
$carrierRules = require __DIR__ . '/carrier-profile.php';

use Symfony\Component\HttpClient\HttpClient;

const XSD_STRING = 'http://www.w3.org/2001/XMLSchema#string';
const RDF_DIR_LANG_STRING = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString';

function resolve_iri(string $base, string $reference): string {
    if (preg_match('/^[A-Za-z][A-Za-z0-9+.-]*:/', $reference)) {
        return $reference;
    }
    $parts = parse_url($base);
    $port = isset($parts['port']) ? ':' . $parts['port'] : '';
    $origin = $parts['scheme'] . '://' . $parts['host'] . $port;
    if (str_starts_with($reference, '#')) {
        return preg_replace('/#.*$/', '', $base) . $reference;
    }
    if (str_starts_with($reference, '/')) {
        return $origin . $reference;
    }
    $path = $parts['path'] ?? '/';
    return $origin . preg_replace('~/[^/]*$~', '/', $path) . $reference;
}

// Fetch and parse the document with the HTML5 DOM parser in PHP 8.4+.
$response = HttpClient::create(['max_redirects' => 5])->request('GET', $argv[1]);
$html = $response->getContent();
$retrievalIri = $response->getInfo('url');
$document = Dom\HTMLDocument::createFromString($html, LIBXML_NOERROR);

// Resolve semantic references against the one canonical source IRI.
$canonical = [];
foreach ($document->querySelectorAll('head link[rel][href]') as $link) {
    $relTokens = preg_split('/\s+/', strtolower($link->getAttribute('rel')));
    if (in_array('canonical', $relTokens, true)) {
        $canonical[] = $link;
    }
}
if (count($canonical) !== 1) {
    throw new RuntimeException('Expected one canonical link');
}
$sourceIri = resolve_iri($retrievalIri, $canonical[0]->getAttribute('href'));

// A normal attribute selector finds every locally complete carrier.
$rows = [];
foreach ($document->querySelectorAll('[rdf-predicate]') as $element) {
    $insideTemplate = false;
    for ($parent = $element->parentElement; $parent !== null; $parent = $parent->parentElement) {
        if ($parent->localName === 'template') {
            $insideTemplate = true;
            break;
        }
    }
    if ($insideTemplate) {
        continue;
    }
    $subject = resolve_iri($sourceIri, $element->getAttribute('rdf-subject'));
    $predicate = resolve_iri($sourceIri, $element->getAttribute('rdf-predicate'));

    // The generated profile identifies any native attribute carrying the value.
    $valueCarriers = array_values(array_filter(
        $carrierRules,
        fn($rule) => $rule[0] === $element->localName && $element->hasAttribute($rule[1])
    ));
    if (count($valueCarriers) > 1) {
        throw new RuntimeException("Conflicting value carriers on <{$element->localName}>");
    }
    $carrier = $valueCarriers[0] ?? null;

    // IRI carrier values are references; resolve them against the semantic source.
    if ($carrier !== null && $carrier[2] === 'iri') {
        $objectIri = resolve_iri($sourceIri, $element->getAttribute($carrier[1]));
        $rows[] = [$subject, $predicate, 'I', $objectIri, '', '', ''];
        continue;
    }

    // Literal carriers are exact strings. Other elements use normalized visible text.
    if ($carrier !== null && $carrier[2] === 'literal') {
        $value = $element->getAttribute($carrier[1]);
    } else {
        $value = trim(preg_replace('/[\t\n\f\r ]+/', ' ', $element->textContent));
    }

    // lang and dir apply only when directly present on this carrier.
    $language = $element->getAttribute('lang') ?? '';
    $rawDirection = strtolower($element->getAttribute('dir') ?? '');
    $direction = in_array($rawDirection, ['ltr', 'rtl'], true) ? $rawDirection : '';

    // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    $datatype = $direction !== '' ? RDF_DIR_LANG_STRING : XSD_STRING;
    if ($element->hasAttribute('rdf-datatype')) {
        $datatype = resolve_iri($sourceIri, $element->getAttribute('rdf-datatype'));
    }
    $rows[] = [$subject, $predicate, 'L', $value, $datatype, $language, $direction];
}
usort($rows, fn($left, $right) => $left[1] <=> $right[1]);
foreach ($rows as $row) {
    echo implode("\t", $row) . "\n";
}

# frozen_string_literal: true

require "faraday"
require "nokogiri"
require "uri"
require_relative "carrier_profile"

XSD_STRING = "http://www.w3.org/2001/XMLSchema#string"
RDF_DIR_LANG_STRING = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

# Fetch and parse the document as HTML5.
response = Faraday.get(ARGV.fetch(0))
raise "HTTP #{response.status}" unless response.success?
retrieval_iri = response.env.url.to_s
document = Nokogiri::HTML5(response.body, url: retrieval_iri)

# Resolve semantic references against the one canonical source IRI.
canonical = document.css("head link[rel][href]").select do |link|
  link["rel"].split.any? { |token| token.casecmp?("canonical") }
end
raise "Expected one canonical link" unless canonical.one?
source_iri = URI.join(retrieval_iri, canonical.first["href"]).to_s

# A normal attribute selector finds every locally complete carrier.
carriers = document.css("[rdf-predicate]").reject do |element|
  element.ancestors("template").any?
end

rows = carriers.map do |element|
  subject = URI.join(source_iri, element["rdf-subject"]).to_s
  predicate = URI.join(source_iri, element["rdf-predicate"]).to_s

  # The generated profile identifies any native attribute carrying the value.
  value_carriers = CARRIER_RULES.select do |rule|
    rule[0] == element.name && element.key?(rule[1])
  end
  raise "Conflicting value carriers on <#{element.name}>" if value_carriers.length > 1
  carrier = value_carriers.first

  # IRI carrier values are references; resolve them against the semantic source.
  if carrier && carrier[2] == "iri"
    object_iri = URI.join(source_iri, element[carrier[1]]).to_s
    [subject, predicate, "I", object_iri, "", "", ""]
  else
    # Literal carriers are exact strings. Other elements use normalized visible text.
    value = if carrier && carrier[2] == "literal"
              element[carrier[1]]
            else
              element.text.gsub(/[\t\n\f\r ]+/, " ").strip
            end

    # lang and dir apply only when directly present on this carrier.
    language = element["lang"] || ""
    raw_direction = (element["dir"] || "").downcase
    direction = %w[ltr rtl].include?(raw_direction) ? raw_direction : ""

    # An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    datatype = direction.empty? ? XSD_STRING : RDF_DIR_LANG_STRING
    if element.key?("rdf-datatype")
      datatype = URI.join(source_iri, element["rdf-datatype"]).to_s
    end
    [subject, predicate, "L", value, datatype, language, direction]
  end
end
rows.sort_by { |row| row[1] }.each do |row|
  puts row.join("\t")
end

import Foundation
#if canImport(FoundationNetworking)
import FoundationNetworking
#endif
import SwiftSoup

let xsdString = "http://www.w3.org/2001/XMLSchema#string"
let rdfDirLangString = "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

func resolve(_ base: String, _ reference: String) throws -> String {
    guard let baseUrl = URL(string: base),
          let result = URL(string: reference, relativeTo: baseUrl)?.absoluteURL else {
        throw URLError(.badURL)
    }
    return result.absoluteString
}

struct HtmlRdfExample {
    static func run() async throws {
        // Fetch and parse the document as HTML.
        let inputURL = URL(string: CommandLine.arguments[1])!
        let (data, response) = try await URLSession.shared.data(from: inputURL)
        guard let http = response as? HTTPURLResponse,
              (200..<300).contains(http.statusCode) else {
            throw URLError(.badServerResponse)
        }
        let retrievalIri = response.url!.absoluteString
        let document = try SwiftSoup.parseHTML(data, retrievalIri)

        // Resolve semantic references against the one canonical source IRI.
        let canonical = try document.select("head link[rel][href]").array().filter { link in
            try link.attr("rel")
                .split(whereSeparator: { $0.isWhitespace })
                .contains { $0.lowercased() == "canonical" }
        }
        guard canonical.count == 1 else {
            throw NSError(
                domain: "HtmlRdfExample",
                code: 1,
                userInfo: [NSLocalizedDescriptionKey: "Expected one canonical link"]
            )
        }
        let sourceIri = try resolve(retrievalIri, canonical[0].attr("href"))

        // A normal attribute selector finds every locally complete carrier.
        var rows: [[String]] = []
        for element in try document.select("[rdf-predicate]").array() {
            let insideTemplate = element.parents().array().contains { $0.tagName() == "template" }
            if insideTemplate { continue }
            let subject = try resolve(sourceIri, element.attr("rdf-subject"))
            let predicate = try resolve(sourceIri, element.attr("rdf-predicate"))

            // The generated profile identifies any native attribute carrying the value.
            let valueCarriers = CarrierProfile.rules.filter { rule in
                rule.element == element.tagName() && element.hasAttr(rule.attribute)
            }
            guard valueCarriers.count <= 1 else {
                throw NSError(
                    domain: "HtmlRdfExample",
                    code: 2,
                    userInfo: [NSLocalizedDescriptionKey:
                        "Conflicting value carriers on <\(element.tagName())>"]
                )
            }
            let carrier = valueCarriers.first

            // IRI carrier values are references; resolve them against the semantic source.
            if let carrier, carrier.kind == "iri" {
                let objectIri = try resolve(sourceIri, element.attr(carrier.attribute))
                rows.append([subject, predicate, "I", objectIri, "", "", ""])
                continue
            }

            // Literal carriers are exact strings. Other elements use normalized text.
            let value: String
            if let carrier, carrier.kind == "literal" {
                value = try element.attr(carrier.attribute)
            } else {
                value = try element.text()
            }

            // lang and dir apply only when directly present on this carrier.
            let language = try element.attr("lang")
            let rawDirection = try element.attr("dir").lowercased()
            let direction = ["ltr", "rtl"].contains(rawDirection) ? rawDirection : ""

            // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
            let datatype: String
            if element.hasAttr("rdf-datatype") {
                datatype = try resolve(sourceIri, element.attr("rdf-datatype"))
            } else {
                datatype = direction.isEmpty ? xsdString : rdfDirLangString
            }
            rows.append([subject, predicate, "L", value, datatype, language, direction])
        }
        for row in rows.sorted(by: { $0[1] < $1[1] }) {
            print(row.joined(separator: "\t"))
        }
    }
}

try await HtmlRdfExample.run()

import 'package:html/dom.dart';
import 'package:html/parser.dart' show parse;
import 'package:html_rdf_library_example/carrier_profile.dart';
import 'package:http/http.dart' as http;

const xsdString = 'http://www.w3.org/2001/XMLSchema#string';
const rdfDirLangString = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString';

bool insideTemplate(Element element) {
  for (var parent = element.parent; parent != null; parent = parent.parent) {
    if (parent.localName == 'template') {
      return true;
    }
  }
  return false;
}

Future<void> main(List<String> args) async {
  // Fetch and parse the document as HTML.
  final response = await http.get(Uri.parse(args[0]));
  if (response.statusCode < 200 || response.statusCode >= 300) {
    throw StateError('HTTP ${response.statusCode}');
  }
  final retrievalIri = response.request!.url;
  final document = parse(response.bodyBytes, sourceUrl: retrievalIri.toString());

  // Resolve semantic references against the one canonical source IRI.
  final canonical = document
      .querySelectorAll('head link[rel][href]')
      .where((link) => link.attributes['rel']!
          .split(RegExp(r'\s+'))
          .any((token) => token.toLowerCase() == 'canonical'))
      .toList();
  if (canonical.length != 1) {
    throw StateError('Expected one canonical link');
  }
  final sourceIri = retrievalIri.resolve(canonical.single.attributes['href']!);

  // A normal attribute selector finds every locally complete carrier.
  final rows = <List<String>>[];
  final carriers = document
      .querySelectorAll('[rdf-predicate]')
      .where((element) => !insideTemplate(element));
  for (final element in carriers) {
    final subject = sourceIri.resolve(element.attributes['rdf-subject']!).toString();
    final predicate = sourceIri.resolve(element.attributes['rdf-predicate']!).toString();

    // The generated profile identifies any native attribute carrying the value.
    final valueCarriers = carrierRules
        .where((rule) => rule.element == element.localName &&
            element.attributes.containsKey(rule.attribute))
        .toList();
    if (valueCarriers.length > 1) {
      throw StateError('Conflicting value carriers on <${element.localName}>');
    }
    final carrier = valueCarriers.isEmpty ? null : valueCarriers.first;

    // IRI carrier values are references; resolve them against the semantic source.
    if (carrier?.kind == 'iri') {
      final objectIri = sourceIri
          .resolve(element.attributes[carrier!.attribute]!)
          .toString();
      rows.add([subject, predicate, 'I', objectIri, '', '', '']);
      continue;
    }

    // Literal carriers are exact strings. Other elements use normalized visible text.
    final value = carrier?.kind == 'literal'
        ? element.attributes[carrier!.attribute]!
        : element.text.replaceAll(RegExp(r'[\t\n\f\r ]+'), ' ').trim();

    // lang and dir apply only when directly present on this carrier.
    final language = element.attributes['lang'] ?? '';
    final rawDirection = (element.attributes['dir'] ?? '').toLowerCase();
    final direction = rawDirection == 'ltr' || rawDirection == 'rtl' ? rawDirection : '';

    // An explicit datatype wins; direction otherwise selects rdf:dirLangString.
    final datatype = element.attributes.containsKey('rdf-datatype')
      ? sourceIri.resolve(element.attributes['rdf-datatype']!).toString()
      : direction.isNotEmpty ? rdfDirLangString : xsdString;
    rows.add([subject, predicate, 'L', value, datatype, language, direction]);
  }
  rows.sort((left, right) => left[1].compareTo(right[1]));
  for (final row in rows) {
    print(row.join('\t'));
  }
}

#include <ctype.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <curl/curl.h>
#include <lexbor/html/html.h>
#include "carrier_profile.h"

#define FIELD 512
#define XSD_STRING "http://www.w3.org/2001/XMLSchema#string"
#define RDF_DIR_LANG_STRING "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

typedef struct {
    char *data;
    size_t length;
} buffer_t;

typedef struct {
    char field[7][FIELD];
} row_t;

typedef struct {
    const char *source;
    row_t rows[8];
    size_t length;
} extraction_t;

static size_t receive(char *data, size_t size, size_t count, void *ctx) {
    buffer_t *buffer = ctx;
    size_t length = size * count;
    char *next = realloc(buffer->data, buffer->length + length + 1);
    if (next == NULL)
        return 0;
    buffer->data = next;
    memcpy(buffer->data + buffer->length, data, length);
    buffer->length += length;
    buffer->data[buffer->length] = '\0';
    return length;
}

static int attribute(lxb_dom_element_t *element, const char *name, char *output) {
    size_t length = 0;
    const lxb_char_t *value = lxb_dom_element_get_attribute(
        element, (const lxb_char_t *)name, strlen(name), &length);
    if (value == NULL) {
        output[0] = '\0';
        return 0;
    }
    if (length >= FIELD)
        length = FIELD - 1;
    memcpy(output, value, length);
    output[length] = '\0';
    return 1;
}

static int tag_is(lxb_dom_element_t *element, const char *expected) {
    size_t length = 0;
    const lxb_char_t *name = lxb_dom_element_local_name(element, &length);
    return strlen(expected) == length && memcmp(name, expected, length) == 0;
}

static void resolve_iri(const char *base, const char *reference, char *output) {
    const char *colon = strchr(reference, ':');
    if (colon != NULL && colon != reference) {
        snprintf(output, FIELD, "%s", reference);
        return;
    }
    if (reference[0] == '#') {
        size_t length = strcspn(base, "#");
        snprintf(output, FIELD, "%.*s%s", (int)length, base, reference);
        return;
    }
    const char *scheme = strstr(base, "://");
    const char *path = scheme == NULL ? base : strchr(scheme + 3, '/');
    size_t origin = path == NULL ? strlen(base) : (size_t)(path - base);
    if (reference[0] == '/') {
        snprintf(output, FIELD, "%.*s%s", (int)origin, base, reference);
        return;
    }
    size_t directory = strlen(base);
    while (directory > origin && base[directory - 1] != '/')
        directory--;
    snprintf(output, FIELD, "%.*s%s", (int)directory, base, reference);
}

static void normalize(const lxb_char_t *input, size_t length, char *output) {
    size_t write = 0;
    int pending_space = 0;
    for (size_t read = 0; read < length && write + 1 < FIELD; read++) {
        unsigned char value = input[read];
        if (value == '\t' || value == '\n' || value == '\f' || value == '\r' ||
            value == ' ') {
            if (write > 0)
                pending_space = 1;
            continue;
        }
        if (pending_space && write + 1 < FIELD)
            output[write++] = ' ';
        pending_space = 0;
        output[write++] = (char)value;
    }
    output[write] = '\0';
}

static void find_canonical(lxb_dom_node_t *node, char *canonical, size_t *count) {
    if (node->type == LXB_DOM_NODE_TYPE_ELEMENT) {
        lxb_dom_element_t *element = lxb_dom_interface_element(node);
        char rel[FIELD], href[FIELD];
        if (tag_is(element, "link") && attribute(element, "rel", rel) &&
            attribute(element, "href", href) && strcasecmp(rel, "canonical") == 0) {
            snprintf(canonical, FIELD, "%s", href);
            (*count)++;
        }
    }
    for (lxb_dom_node_t *child = node->first_child; child != NULL; child = child->next)
        find_canonical(child, canonical, count);
}

static void collect(lxb_dom_node_t *node, extraction_t *ctx) {
    int template = 0;
    if (node->type == LXB_DOM_NODE_TYPE_ELEMENT) {
        lxb_dom_element_t *element = lxb_dom_interface_element(node);
        template = tag_is(element, "template");
        char predicate_ref[FIELD];
        if (attribute(element, "rdf-predicate", predicate_ref)) {
            row_t *row = &ctx->rows[ctx->length++];
            char subject_ref[FIELD], carrier_value[FIELD], datatype_ref[FIELD], language[FIELD],
                direction[FIELD];
            attribute(element, "rdf-subject", subject_ref);
            resolve_iri(ctx->source, subject_ref, row->field[0]);
            resolve_iri(ctx->source, predicate_ref, row->field[1]);

            /* The generated profile identifies a native attribute carrying the value. */
            const carrier_rule_t *carrier = NULL;
            for (size_t i = 0; i < CARRIER_RULE_COUNT; i++) {
                if (tag_is(element, CARRIER_RULES[i].element) &&
                    attribute(element, CARRIER_RULES[i].attribute, carrier_value)) {
                    if (carrier != NULL)
                        abort();
                    carrier = &CARRIER_RULES[i];
                }
            }

            /* IRI carrier values are references resolved against the semantic source. */
            if (carrier != NULL && strcmp(carrier->kind, "iri") == 0) {
                strcpy(row->field[2], "I");
                resolve_iri(ctx->source, carrier_value, row->field[3]);
                row->field[4][0] = row->field[5][0] = row->field[6][0] = '\0';
            } else {
                strcpy(row->field[2], "L");

                /* Literal carriers are exact; other elements use normalized text. */
                if (carrier != NULL && strcmp(carrier->kind, "literal") == 0) {
                    strcpy(row->field[3], carrier_value);
                } else {
                    size_t length = 0;
                    lxb_char_t *text = lxb_dom_node_text_content(node, &length);
                    normalize(text, length, row->field[3]);
                }

                /* lang and dir apply only when directly present on this carrier. */
                attribute(element, "lang", language);
                strcpy(row->field[5], language);
                attribute(element, "dir", direction);
                for (char *p = direction; *p; p++)
                    *p = (char)tolower((unsigned char)*p);
                if (strcmp(direction, "ltr") != 0 && strcmp(direction, "rtl") != 0)
                    direction[0] = '\0';
                strcpy(row->field[6], direction);

                /* Explicit datatype wins; direction otherwise means dirLangString. */
                if (attribute(element, "rdf-datatype", datatype_ref))
                    resolve_iri(ctx->source, datatype_ref, row->field[4]);
                else
                    strcpy(row->field[4], direction[0] ? RDF_DIR_LANG_STRING : XSD_STRING);
            }
        }
    }
    if (!template)
        for (lxb_dom_node_t *child = node->first_child; child != NULL; child = child->next)
            collect(child, ctx);
}

static int compare_rows(const void *left, const void *right) {
    return strcmp(((const row_t *)left)->field[1], ((const row_t *)right)->field[1]);
}

int main(int argc, char **argv) {
    if (argc != 2)
        return EXIT_FAILURE;

    /* Fetch and parse the document as HTML. */
    CURL *curl = curl_easy_init();
    buffer_t html = {0};
    curl_easy_setopt(curl, CURLOPT_URL, argv[1]);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, receive);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &html);
    if (curl_easy_perform(curl) != CURLE_OK)
        return EXIT_FAILURE;
    char *retrieval = NULL;
    curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &retrieval);
    lxb_html_document_t *document = lxb_html_document_create();
    if (document == NULL || lxb_html_document_parse(document, (lxb_char_t *)html.data,
                                                    html.length) != LXB_STATUS_OK)
        return EXIT_FAILURE;

    /* Resolve semantic references against the one canonical source IRI. */
    char canonical[FIELD];
    size_t canonical_count = 0;
    find_canonical(lxb_dom_interface_node(document), canonical, &canonical_count);
    if (canonical_count != 1)
        return EXIT_FAILURE;
    char source[FIELD];
    resolve_iri(retrieval, canonical, source);

    /* Walk the tree to find every locally complete statement carrier. */
    extraction_t extraction = {.source = source, .length = 0};
    collect(lxb_dom_interface_node(document), &extraction);
    qsort(extraction.rows, extraction.length, sizeof(row_t), compare_rows);
    for (size_t i = 0; i < extraction.length; i++) {
        for (size_t field = 0; field < 7; field++)
            printf("%s%c", extraction.rows[i].field[field], field == 6 ? '\n' : '\t');
    }
    lxb_html_document_destroy(document);
    curl_easy_cleanup(curl);
    free(html.data);
    return EXIT_SUCCESS;
}

#include <algorithm>
#include <cctype>
#include <cstring>
#include <iostream>
#include <stdexcept>
#include <string>
#include <string_view>
#include <vector>
#include <curl/curl.h>
#include <lexbor/html/html.h>
#include "carrier_profile.hpp"

using Row = std::vector<std::string>;
constexpr auto XSD_STRING = "http://www.w3.org/2001/XMLSchema#string";
constexpr auto RDF_DIR_LANG_STRING =
    "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString";

static size_t receive(char *data, size_t size, size_t count, void *ctx) {
    static_cast<std::string *>(ctx)->append(data, size * count);
    return size * count;
}

static std::string attr(lxb_dom_element_t *element, const char *name) {
    size_t length = 0;
    auto value = lxb_dom_element_get_attribute(
        element, reinterpret_cast<const lxb_char_t *>(name), std::strlen(name), &length);
    return value == nullptr ? std::string()
                            : std::string(reinterpret_cast<const char *>(value), length);
}

static bool has_attr(lxb_dom_element_t *element, const char *name) {
    return lxb_dom_element_has_attribute(
        element, reinterpret_cast<const lxb_char_t *>(name), std::strlen(name));
}

static bool tag_is(lxb_dom_element_t *element, std::string_view expected) {
    size_t length = 0;
    auto name = lxb_dom_element_local_name(element, &length);
    return expected == std::string_view(reinterpret_cast<const char *>(name), length);
}

static std::string resolve(std::string base, const std::string &reference) {
    auto colon = reference.find(':');
    if (colon != std::string::npos && colon != 0)
        return reference;
    if (reference.starts_with('#')) {
        auto fragment = base.find('#');
        if (fragment != std::string::npos)
            base.resize(fragment);
        return base + reference;
    }
    auto scheme = base.find("://");
    auto slash = base.find('/', scheme == std::string::npos ? 0 : scheme + 3);
    auto origin = slash == std::string::npos ? base.size() : slash;
    if (reference.starts_with('/'))
        return base.substr(0, origin) + reference;
    return base.substr(0, base.rfind('/') + 1) + reference;
}

static std::string normalize(const lxb_char_t *value, size_t length) {
    std::string output;
    bool pending = false;
    for (size_t i = 0; i < length; i++) {
        unsigned char c = value[i];
        bool space = c == '\t' || c == '\n' || c == '\f' || c == '\r' || c == ' ';
        if (space) {
            pending = !output.empty();
            continue;
        }
        if (pending)
            output += ' ';
        pending = false;
        output += static_cast<char>(c);
    }
    return output;
}

static void find_canonical(lxb_dom_node_t *node, std::vector<std::string> &canonicals) {
    if (node->type == LXB_DOM_NODE_TYPE_ELEMENT) {
        auto element = lxb_dom_interface_element(node);
        if (tag_is(element, "link") && attr(element, "rel") == "canonical") {
            canonicals.push_back(attr(element, "href"));
        }
    }

    for (auto child = node->first_child; child != nullptr; child = child->next) {
        find_canonical(child, canonicals);
    }
}

static void collect(lxb_dom_node_t *node, const std::string &source,
                    std::vector<Row> &rows) {
    bool inside_template = false;
    if (node->type == LXB_DOM_NODE_TYPE_ELEMENT) {
        auto element = lxb_dom_interface_element(node);
        inside_template = tag_is(element, "template");

        if (has_attr(element, "rdf-predicate")) {
            auto subject = resolve(source, attr(element, "rdf-subject"));
            auto predicate = resolve(source, attr(element, "rdf-predicate"));

            // The generated profile identifies a native attribute carrying the value.
            const CarrierRule *carrier = nullptr;
            for (const auto &rule : CARRIER_RULES) {
                if (tag_is(element, rule.element) && has_attr(element, rule.attribute.data())) {
                    if (carrier != nullptr) {
                        throw std::runtime_error("Conflicting value carriers");
                    }
                    carrier = &rule;
                }
            }

            // IRI carrier values are references resolved against the semantic source.
            if (carrier != nullptr && carrier->kind == "iri") {
                auto object = resolve(source, attr(element, carrier->attribute.data()));
                rows.push_back({subject, predicate, "I", object, "", "", ""});
            } else {
                // Literal carriers are exact; other elements use normalized text.
                std::string value;
                if (carrier != nullptr && carrier->kind == "literal") {
                    value = attr(element, carrier->attribute.data());
                } else {
                    size_t length = 0;
                    auto text = lxb_dom_node_text_content(node, &length);
                    value = normalize(text, length);
                }

                // lang and dir apply only when directly present on this carrier.
                auto language = attr(element, "lang");
                auto direction = attr(element, "dir");
                std::ranges::transform(direction, direction.begin(),
                                       [](unsigned char c) { return std::tolower(c); });
                if (direction != "ltr" && direction != "rtl") {
                    direction.clear();
                }

                // Explicit datatype wins; direction otherwise selects dirLangString.
                std::string datatype = direction.empty() ? XSD_STRING : RDF_DIR_LANG_STRING;
                if (has_attr(element, "rdf-datatype")) {
                    datatype = resolve(source, attr(element, "rdf-datatype"));
                }
                rows.push_back(
                    {subject, predicate, "L", value, datatype, language, direction});
            }
        }
    }

    if (!inside_template) {
        for (auto child = node->first_child; child != nullptr; child = child->next) {
            collect(child, source, rows);
        }
    }
}

int main(int argc, char **argv) {
    if (argc != 2) {
        return 1;
    }

    // Fetch and parse the document as HTML.
    std::string html;
    auto curl = curl_easy_init();
    curl_easy_setopt(curl, CURLOPT_URL, argv[1]);
    curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1L);
    curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, receive);
    curl_easy_setopt(curl, CURLOPT_WRITEDATA, &html);
    if (curl_easy_perform(curl) != CURLE_OK) {
        return 1;
    }
    char *retrieval = nullptr;
    curl_easy_getinfo(curl, CURLINFO_EFFECTIVE_URL, &retrieval);
    auto document = lxb_html_document_create();
    if (document == nullptr ||
        lxb_html_document_parse(document, reinterpret_cast<const lxb_char_t *>(html.data()),
                                html.size()) != LXB_STATUS_OK) {
        return 1;
    }

    // Resolve semantic references against the one canonical source IRI.
    std::vector<std::string> canonicals;
    find_canonical(lxb_dom_interface_node(document), canonicals);
    if (canonicals.size() != 1) {
        throw std::runtime_error("Expected one canonical link");
    }
    auto source = resolve(retrieval, canonicals.front());

    // Walk the tree to find every locally complete statement carrier.
    std::vector<Row> rows;
    collect(lxb_dom_interface_node(document), source, rows);
    std::ranges::sort(rows, {}, [](const Row &row) { return row[1]; });

    for (const auto &row : rows) {
        for (size_t i = 0; i < row.size(); i++) {
            std::cout << row[i] << (i + 1 == row.size() ? '\n' : '\t');
        }
    }
    lxb_html_document_destroy(document);
    curl_easy_cleanup(curl);
    return 0;
}

defmodule HtmlRdfExample do
  @xsd_string "http://www.w3.org/2001/XMLSchema#string"
  @rdf_dir_lang_string "http://www.w3.org/1999/02/22-rdf-syntax-ns#dirLangString"

  def main([url]) do
    # Fetch and parse the document as HTML5.
    response = Req.get!(url)
    {:ok, document} = Floki.parse_document(response.body, html_parser: Floki.HTMLParser.Html5ever)

    # Resolve semantic references against the one canonical source IRI.
    canonical =
      document
      |> Floki.find("head link[rel][href]")
      |> Enum.filter(fn {_tag, attrs, _children} ->
        attrs
        |> attr("rel")
        |> String.split()
        |> Enum.any?(fn token -> String.downcase(token) == "canonical" end)
      end)

    if length(canonical) != 1 do
      raise "Expected one canonical link"
    end

    {_tag, canonical_attrs, _children} = hd(canonical)
    source_iri = resolve(url, attr(canonical_attrs, "href"))

    # Attribute-name discovery finds every locally complete carrier.
    document
    |> collect_statements()
    |> Enum.map(&row(&1, source_iri))
    |> Enum.sort_by(&Enum.at(&1, 1))
    |> Enum.each(&IO.puts(Enum.join(&1, "\t")))
  end

  defp collect_statements(nodes) when is_list(nodes) do
    Enum.flat_map(nodes, &collect_statements/1)
  end

  defp collect_statements({"template", _attrs, _children}), do: []

  defp collect_statements({tag, attrs, children} = node) when is_binary(tag) do
    own = if attr(attrs, "rdf-predicate") == "", do: [], else: [node]
    own ++ collect_statements(children)
  end

  defp collect_statements(_node), do: []

  defp row({tag, attrs, _children} = node, source_iri) do
    subject = resolve(source_iri, attr(attrs, "rdf-subject"))
    predicate = resolve(source_iri, attr(attrs, "rdf-predicate"))

    # The generated profile identifies any native attribute carrying the value.
    carriers =
      CarrierProfile.rules()
      |> Enum.filter(fn {element, attribute, _kind, _operation} ->
        element == tag and has_attr?(attrs, attribute)
      end)

    if length(carriers) > 1 do
      raise "Conflicting value carriers on <#{tag}>"
    end

    carrier = List.first(carriers)

    # IRI carrier values are references; resolve them against the semantic source.
    if carrier != nil and elem(carrier, 2) == "iri" do
      attribute = elem(carrier, 1)
      [subject, predicate, "I", resolve(source_iri, attr(attrs, attribute)), "", "", ""]
    else
      # Literal carriers are exact strings. Other elements use normalized visible text.
      value =
        if carrier != nil and elem(carrier, 2) == "literal" do
          attr(attrs, elem(carrier, 1))
        else
          node |> Floki.text() |> normalize()
        end

      # lang and dir apply only when directly present on this carrier.
      language = attr(attrs, "lang")
      raw_direction = attrs |> attr("dir") |> String.downcase()
      direction = if raw_direction in ["ltr", "rtl"], do: raw_direction, else: ""

      # An explicit datatype wins; direction otherwise selects rdf:dirLangString.
      datatype =
        cond do
          attr(attrs, "rdf-datatype") != "" -> resolve(source_iri, attr(attrs, "rdf-datatype"))
          direction != "" -> @rdf_dir_lang_string
          true -> @xsd_string
        end

      [subject, predicate, "L", value, datatype, language, direction]
    end
  end

  defp attr(attrs, name) do
    Enum.find_value(attrs, "", fn {key, value} ->
      if key == name, do: value
    end)
  end

  defp has_attr?(attrs, name), do: Enum.any?(attrs, fn {key, _value} -> key == name end)

  defp normalize(value), do: value |> String.replace(~r/[\t\n\f\r ]+/, " ") |> String.trim()
  defp resolve(base, reference), do: base |> URI.parse() |> URI.merge(reference) |> to_string()
end