chore(cli): fetch programming languages names from wikidata

Author: elf-pavlik

catalog-data.ttl

@@ -4,6 +4,8 @@
 @prefix owl: <http://www.w3.org/2002/07/owl#> .
 @prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
 @prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#> .
+@prefix wd: <http://www.wikidata.org/entity/> .
+@prefix wdt: <http://www.wikidata.org/prop/direct/> .
 @prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
 
 <http://harth.org/andreas/foaf#ah> a ex:Person ;
@@ -13,6 +15,57 @@
   ex:name "Andreas Harth" ;
   ex:webid <http://harth.org/andreas/foaf#ah> .
 
+wd:Q161053 ex:name "Ruby" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q17147733 ex:name "Swift" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q2005 ex:name "JavaScript" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q2370 ex:name "C#" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q251 ex:name "Java" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q28865 ex:name "Python" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q34010 ex:name "Haskell" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q37227 ex:name "Go" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q3816639 ex:name "Kotlin" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q406009 ex:name "Dart" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q42478 ex:name "Perl" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q460584 ex:name "Scala" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q51885456 ex:name "Zig" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q5362035 ex:name "Elixir" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q575650 ex:name "Rust" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q59 ex:name "PHP" ;
+  wdt:P31 wd:Q9143 .
+
+wd:Q978185 ex:name "TypeScript" ;
+  wdt:P31 wd:Q9143 .
+
 <https://45h.inrupt.net/profile/card#me> a ex:Person ;
   ex:name "Luc" ;
   ex:webid <https://45h.inrupt.net/profile/card#me> .

cli/aggregations/wikidata.ts

@@ -0,0 +1,47 @@
+import { DataFactory, Store } from 'n3'
+import { dereferenceToStore } from 'rdf-dereference-store'
+import { queryDataset, prefixes, ex, schema } from '../util.ts'
+import type { NamedNode } from '@rdfjs/types'
+
+export async function aggregateWikidata(dataset: Store): Promise<Store> {
+  return aggregateProgrammingLanguages(dataset)
+}
+
+async function getName(store: Store, language: NamedNode, tag: string): Promise<string | undefined> {
+  const query = `
+      SELECT ?s ?name
+      WHERE {
+        <${language.value}> <${schema.name}> ?name
+        FILTER ( LANG(?name) = "${tag}" )
+      }`
+
+  let bindings = await queryDataset(store, query)
+  return bindings[0]?.get('name')?.value
+}
+
+export async function aggregateProgrammingLanguages(dataset: Store): Promise<Store> {
+  const instanceOf = `${prefixes.wdt}P31`
+  const ProgrammingLanguage = `${prefixes.wd}Q9143`
+  const langQuery = `
+    SELECT ?s ?name
+    WHERE {
+      ?s <${instanceOf}> <${ProgrammingLanguage}> .
+      OPTIONAL { ?s <${ex.name}> ?name . }
+    }`
+  const lbindings = await queryDataset(dataset, langQuery)
+  const withoutNames = lbindings.filter(b => !b.get('name')).map(b => b.get('s') as NamedNode)
+  console.info(`Fetching names for programming langages: ${withoutNames.length}`)
+  for (const language of withoutNames) {
+    const { store } = await dereferenceToStore(language.value.replace('http', 'https'))
+    let name = await getName(store, language, 'en')
+    if (!name) name = await getName(store, language, 'mul')
+    if (!name) {
+      console.warn(`Couldn't find name for ${language.value}`)
+      continue
+    }
+    const quad = DataFactory.quad(language, ex.terms.name, DataFactory.literal(name))
+    dataset.add(quad)
+  }
+  return dataset
+}
+

cli/index.ts

@@ -6,6 +6,7 @@ import { validateWebid } from './validations/webid.ts'
 import { migrateWebid } from './migrations/webid.ts'
 import { aggregateW3C } from './aggregations/w3c.ts'
 import { aggregateGithub } from './aggregations/github.ts'
+import { aggregateWikidata } from './aggregations/wikidata.ts'
 
 const dataPath = getPath(import.meta.url, '../catalog-data.ttl')
 const dataset = await loadData(dataPath)
@@ -58,4 +59,12 @@ aggregate.command('github')
     await saveData(updated, dataPath)
   })
 
+aggregate.command('wikidata')
+  .description('Adds data from Wikidata')
+  .action(async () => {
+    console.info('Fetching data Wikiadta')
+    const updated = await aggregateWikidata(dataset)
+    await saveData(updated, dataPath)
+  })
+
 program.parse(process.argv)

cli/util.ts

@@ -10,21 +10,16 @@ import { QueryEngine } from '@comunica/query-sparql-rdfjs'
 import { Store } from 'n3'
 import { write } from '@jeswr/pretty-turtle'
 
-const prefixes = {
+export const prefixes = {
   rdf: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#',
   xsd: 'http://www.w3.org/2001/XMLSchema#',
+  wd: 'http://www.wikidata.org/entity/',
+  wdt: 'http://www.wikidata.org/prop/direct/',
   con: 'https://solidproject.solidcommunity.net/catalog/taxonomy#',
   cdata: 'https://solidproject.solidcommunity.net/catalog/data#',
   ex: 'http://example.org#',
 }
 
-export async function formatData(filePath: string): Promise<void> {
-  const fromStream = await readQuadStream(filePath)
-  const fromQuads = await arrayifyStream(fromStream)
-  const outString = await write(fromQuads, { prefixes, ordered: true })
-  fs.writeFileSync(filePath, outString)
-}
-
 export async function loadData(filePath: string): Promise<Store> {
   const fromStream = await readQuadStream(filePath)
   return new Store(await arrayifyStream(fromStream))
@@ -35,7 +30,9 @@ export async function saveData(dataset: Store, filePath: string): Promise<void>
   fs.writeFileSync(filePath, outString)
 }
 
-export const ex = createVocabulary('http://example.org#', 'webid', 'siloId', 'member', 'siloUsername', 'Person', 'Organization')
+export const ex = createVocabulary('http://example.org#', 'name', 'webid', 'siloId', 'member', 'siloUsername', 'Person', 'Organization')
+export const schema = createVocabulary('http://schema.org/', 'name')
+export const rdfs = createVocabulary('http://www.w3.org/2000/01/rdf-schema#', 'label')
 
 export function getPath(from: string, to: string): string {
   const __filename = fileURLToPath(from)