Change endpoints to Commons for long term resilience

[hugolpz]

Objective

SignIt does 2 external queries at start sw.js#L43-L47. Since Lingualibre.org will be phased out, migration to Wikidata Query Service's endpoint (WDQS) is required.

Variable name	Role	Current endpoint	Wanted endpoint
sparqlSignLanguagesQuery	List activated sign language	LinguaLibre	WDQS
sparqlSignVideosQuery	Given a category name, list videos in it	Lingualibre	WDQS

A third query sw.js#L48-L63is not actually used in the code.

1. sparqlSignLanguagesQuery

// Needs : 
var signLanguage = { wikidata: "" , label: "", iso: "" };

Current LL query

Below, the prettyfied SPARQL of the current query, with lang = Q99628 :

SELECT ?id ?idLabel ?wikidata
WHERE {
 ?id prop:P2 entity:Q4 .
 ?id prop:P24 entity:Q88890 .
 ?id prop:P12 ?wikidata
 SERVICE wikibase:label { bd:serviceParam wikibase:language "fr,en". }
}

Replacement Wikidata query

Last exploration was this (WDQS demo) :

SELECT ?signLanguage ?LinguaLibreID ?practicians ?iso ?signLanguageLabel ?signLanguageNative
WHERE {
  SERVICE wikibase:label { bd:serviceParam wikibase:language "[AUTO_LANGUAGE],en". }
  ?signLanguage wdt:P31 wd:Q34228;
    wdt:P10369 ?LinguaLibreID.
  OPTIONAL { ?signLanguage wdt:P1098 ?practicians. }
  OPTIONAL { ?signLanguage wdt:P1705 ?signLanguageNative. }
  OPTIONAL { ?signLanguage wdt:P220 ?iso. }
}

2. sparqlSignVideosQuery

var word, filename, speaker;

Current LL query

Below, the prettyfied SPARQL of the current query, with lang = Q99628 :

SELECT ?word ?filename ?speaker 
WHERE { 
  ?record prop:P2 entity:Q2 . 
  ?record prop:P4 entity:Q99628. 
  ?record prop:P7 ?word . 
  ?record prop:P3 ?filename .      
  ?record prop:P5 ?speakerItem . 
  ?speakerItem rdfs:label ?speaker filter ( lang( ?speaker ) = "en" ) .
  }

Replacement query

• Word will have to be extracted via string clean up from the title or filename, for now.
• Speaker will disappear, for now.
• Refined query + data migration to Commons later may bring those back.

SELECT ?title ?filename ?entity
WHERE {
  VALUES ?categories { 
    "Category:Lingua_Libre_pronunciation-inl"
  }
  SERVICE wikibase:mwapi {
    bd:serviceParam wikibase:api "Generator" ;
                    wikibase:endpoint "commons.wikimedia.org" ;
                    mwapi:gcmtitle ?categories ;
                    mwapi:generator "categorymembers" ;
                    mwapi:gcmtype "file" ;
                    mwapi:gcmlimit "max" .
    ?title wikibase:apiOutput mwapi:title .
    ?pageid wikibase:apiOutput "@pageid" .
  }
  BIND (URI(CONCAT('https://commons.wikimedia.org/entity/M', ?pageid)) AS ?entity)
  BIND (URI(CONCAT('https://commons.wikimedia.org/wiki/', ?title)) AS ?filename)
}

Alternative API query

/* ************************************************************ */
/* Fetch category members via API query *********************** */
var fetchCatURL = function(name){ 
  var params = {
    action: "query",
    list: "categorymembers",
    cmtitle: "Category:" + name,
    cmnamespace: 6,
    cmtype: "file",
    cmlimit: "max"
  };
  if (cont) {
    params.continue = cont.continue;
    params.cmcontinue = cont.cmcontinue;
  }
  const queryString = new URLSearchParams(params).toString();
	return `https://commons.wikimedia.org/w/api.php?${queryString}&format=json&origin=*`;
}
var fetchCatMembers = function(name, cont) {
  var apiUrl = fetchCatURL(name);
  var membersConcat = [];
  fetch(apiUrl).then(response => {
      if (!response.ok) {
        throw new Error('Network response was not ok');
      }
      return response.json();
    })
    .then(data => {
      var members = data.query.categorymembers;
      membersConcat =[ ...membersConcat,...members ];
      console.log("API url: ", members)
      // If continue token, recursive fetch
      if (data.continue) { 
        fetchCatMembers(name, data.continue);
      }
      console.log(membersConcat.length)
    }).catch(error => {
      console.error("API request failed:", error);
    });
    return membersConcat;
}
var categoryName = "Videos Langue des signes française";
var categoryMembersFull = fetchCatMembers(catName, undefined);

Filename to filepath url

See https://github.com/hugolpz/NamesOfTheLand/blob/main/index.html#L64-L87

Suggested JS

Base API calls :

• https://commons.wikimedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:Lingua_Libre_pronunciation-fsl&cmlimit=500&format=json&origin=*
• https://commons.wikimedia.org/w/api.php?action=query&list=categorymembers&cmtype=file&cmtitle=Category:Lingua_Libre_pronunciation-fsl&cmlimit=500&format=json&origin=*

Listing media :

• https://lingualibre.org/wiki/Help:APIs#See_also
• https://jsfiddle.net/hugolpz/7fkdsmu5/
• https://jsfiddle.net/6rbcu1yw/2/
  • [classic] Query on Commons:Category:Lingual Libre pronuncitation-fsl > word: crabe, sorted by most relevant
  • [regex] Query on Commons:Category:Lingual Libre pronuncitation-fsl > word: crabe, sorted by most relevant
• Input : Commons API URL + iso639-3 + word
• Response: List of relevant Commons Lingua Libre audio files

Query on individual file

• WDQS > WCQS > Given category name, return members files with Media ids and filenames
• Wikimedia Commons > API > Given file Media id, return P9533 transcription
• Wikimedia Commons > API > Given file Title, return P9533 transcription

[ishqDehlvi]

Hey there can I try this issue?? as far as I am getting the issue is about the discrepancies with the language parameters as well as the queries. lack of query import too I guess. Am I correct??

[hugolpz]

Hello @ishqDehlvi ,
@kabir-afk is currently in a summer internship on SignIt so he has the priority to allow him to experiences the whole web extension's challenges. I will discuss this with him. What I can let you know is we will need lightly involved volunteers this Automn 2024 to review and maintain the enhanced code currently committed by Kabir. As Kabir will fly to new adventures by then. I will still be around to guide the new comers.

[hugolpz]

File full path

const fileUrl = "https://commons.wikimedia.org/wiki/Special:FilePath/" + encodeURIComponent(fileData.title);

or

function commonsFileUrl(filename, fileFormat = null, width = 0) {
    const hashedFilename = CryptoJS.MD5(filename).toString();
    const baseUrl = "https://upload.wikimedia.org/wikipedia/commons";
    let path;
    // Images
    if (width !== 0) {
        path = `thumb/${hashedFilename[0]}/${hashedFilename.slice(0, 2)}/${filename}/${width}px-${filename}`;
        if (filename.slice(-4).toLowerCase() === '.webm') {
            path += ".png";
        }
    } 
    // Audios, videos & others
    else if (fileFormat !== null) {
        path = `transcoded/${hashedFilename[0]}/${hashedFilename.slice(0, 2)}/${filename}/${filename}.${fileFormat}`;
    } else {
        path = `${hashedFilename[0]}/${hashedFilename.slice(0, 2)}/${filename}`;
    }

    return `${baseUrl}/${path}`;
}

function escape (str){
    return str.replace(/%20| /g,'_').replace(/\(/g,'%28').replace(/\)/g,'%29').replace(/\'/g,'%27')
}

function pathToFilename(filepath){
 	let filenameParts = filepath.split('/'),
    	filename = decodeURI(filenameParts[filenameParts.length-1]) ;
  console.log("pathToFilename(): ", { filepath, filename })
return filename;
}