diff options
| -rw-r--r-- | README | 70 | ||||
| -rw-r--r-- | example-sparql.scm | 48 | ||||
| -rw-r--r-- | wikidata/apis.scm (renamed from wikidata.scm) | 95 | ||||
| -rw-r--r-- | wikidata/apis.scm~ | 290 | ||||
| -rw-r--r-- | wikidata/sparql.scm | 111 |
5 files changed, 571 insertions, 43 deletions
@@ -9,26 +9,84 @@ It is inspired by guile-sparql, python-wikidata_guessing and python-Wikidata. It currently has the following dependencies besides guile: * guile-json -* guix (only the 2 procedures in (guix import json) for alist functionality. +* guix 0.15 or later + * (guix import json) 2 procedures in for alist functionality and + (guix import utils) and (guix http-client) for fetching from + servers with error reporting. + +(NOTE: We should really factor out a lot of the generic code in guix in +external libraries to keep libraries like this one light-weight.) ## Use as module It is exported as "wikidata". It can be imported with: ``` -(use-modules (wikidata)) +(use-modules (wikidata apis)) ; for search and claims APIs ``` +And: +``` +(use-modules (wikidata sparql)) ; for support for SPARQL queries +``` + +The first enables you to get all the information in any Q-article in +Wikidata. + +The second one is very powerfull and enables you e.g. to construct a +query to get a list of all free software (Q341) in Wikidata. This +could then e.g. be automatically correlated with package data in guix +adding a field with the Q-id to the relevant package. ## Example usage +### Show + +To search for paris and get the first 15 results with sv labels +and descriptions, do: + +``` +(search "paris" #:limit 15 #:language 'sv) +``` + +#### Notes: + +Fallback language is 'en and output is currently limited to Qid, +label, description (Use the medium level procedures to get other +data). No more than 50 (500 for bots) results are allowed. + +There is a bug in the API so only english results are returned at the +time of this writing (dec 2018). + +#### Example output: + +First 15: Label & Description +Q90: Paris: capital and largest city of Fr... +Q167646: Paris: son of Priam, king of Troy +Q830149: Paris: county seat of Lamar County, T... +Q3181341: Paris: county seat of Bourbon County,... +Q162121: Paris: genus of plants +Q576584: Paris: city in Illinois +Q1018504: Paris: city in Tennessee, United Stat... +Q984459: Paris: city in Idaho +Q79917: Paris: city in Arkansas +Q160946: Paris, Texas: 1984 film by Wim Wenders +Q934294: Paris: town in Maine, USA +Q18331346: Paris: family name +Q960025: Paris: city in Missouri +Q1158980: Paris: male given name +Q366081: Paris Bordone: Italian artist + + +### Medium level-example: extract-search + To search on wikidata and get back an alist with the 10 first results with each element being an alist of label, desc, qid, do this: ``` -(extract-result "Guix") +(extract-search "Guix") ``` -Example output: +#### Example output: scheme@(guile-user) [2]> (load "wikidata.scm") scheme@(guile-user) [2]> ,use(wikidata) @@ -54,4 +112,6 @@ Longyu") ("id" . "Q8172534")) (("label" . "Guixia Zhao") ("id" . "Q56528602")) (("label" . "Guixue Wang") ("description" . #f) ("id" . "Q45902385"))) - +You could then map over this list and get the label and description or +author(s) from Wikidata via queries to the wbgetentity API. See the +source for procedures. diff --git a/example-sparql.scm b/example-sparql.scm new file mode 100644 index 0000000..4bfd75a --- /dev/null +++ b/example-sparql.scm @@ -0,0 +1,48 @@ +;;; Copyright © 2017 Roel Janssen <roel@gnu.org> +;;; Copyright © 2018 swedebugia <swedebugia@riseup.net> +;;; +;;; This file is part of guile-wikidata. +;;; +;;; guile-wikidata is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; guile-wikidata is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with guile-wikidata. If not, see <http://www.gnu.org/licenses/>. + +;; The following step is not needed with Guix: +;; We must add the (wikidata sparql) module to GNU Guile's load path +;; before we can load it. This means we have to add the directory +;; that leads to the sparql/ directory to the %load-path variable. +; (add-to-load-path "/path/to/the/root/of/the/repository") + +;; We need the following modules to show the query. +(use-modules (ice-9 format) + (wikidata sparql)) + +;; Example query passed to the screen for you to eyeball +(show-sparql + ;; FIXME why is LIMIT not honored? + (format #f " +PREFIX wd: <http://www.wikidata.org/entity/> +PREFIX wdt: <http://www.wikidata.org/prop/direct/> +SELECT DISTINCT ?item +WHERE \\{?item wdt:P31/wdt:P279* wd:Q19723451\\} +LIMIT 10 +")) + +;; See more example queries at +;; https://www.wikidata.org/wiki/Wikidata:SPARQL_query_service/queries/examples + +;;; To generate SPARQL queries I recommend +;;; https://query.wikidata.org/. It is excellent because it looks up +;;; all the P- and Q-ids for you and generate the SPARQL for you to +;;; insert above. :D + + diff --git a/wikidata.scm b/wikidata/apis.scm index 8931500..ac77cf6 100644 --- a/wikidata.scm +++ b/wikidata/apis.scm @@ -15,17 +15,24 @@ ;;; You should have received a copy of the GNU General Public License ;;; along with guile-wikidata. If not, see <http://www.gnu.org/licenses/>. -;;; See -;;; https://www.mediawiki.org/wiki/API:Presenting_Wikidata_knowledge -;;; for a good description of workflow when integrating Wikidata into -;;; an application. +;; See +;; https://www.mediawiki.org/wiki/API:Presenting_Wikidata_knowledge +;; for a good description of workflow when integrating the Wikidata API into +;; an application. -(define-module (wikidata) +;; See the example in the bottom for how to use this library. + +(define-module (wikidata apis) #:use-module (ice-9 format) #:use-module (ice-9 optargs) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) #:use-module (json) + #:use-module (guix http-client) + #:use-module (guix import utils) #:use-module (guix import json) #:use-module (srfi srfi-1) + #:use-module (srfi srfi-34) #:use-module (web uri) #:export (show)) @@ -35,34 +42,32 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;TODO implement caching -(define (wdquery uri) +(define (wdquery-alist uri) "Fetch the data, return an alist" (json-fetch-alist uri)) ;; Inspired by PYPI wikidata_suggest (define* (search-uri name #:key - (format "json") - (language "en") - (type "item") - (continue "0") - (limit "10")) + (format 'json) + (language 'en) + (type 'item) + (continue 0) + (limit 'limit)) "Build URI for the Wikidata wbsearchentities API." (let ((url "https://www.wikidata.org/w/api.php") (& "&") - (= "=") - (action "wbsearchentities")) + (= "=")) (string->uri (string-append url "?" "search" = name & - "action" = action & - "format" = format & - "language" = language & - "type" = type & - "continue" = continue & - "limit" = limit - )))) + "action" = "wbsearchentities" & + "format" = (symbol->string format) & + "language" = (symbol->string language) & + "type" = (symbol->string type) & + "continue" = (number->string continue) & + "limit" = (number->string limit))))) ;; Inspired by ;; https://opendata.stackexchange.com/questions/5248/how-to-get-the-name-of-a-wikidata-item @@ -71,7 +76,9 @@ #:optional property #:key (language 'en) (format 'json)) - "Build URI for the Wikidata wbsearchintities API." + "Build URI for the Wikidata wbsearchintities API. PROPERTY is a +string containing one of the props here: +https://www.wikidata.org/wiki/Special:ApiHelp/wbgetentities" (let* ((url "https://www.wikidata.org/w/api.php") (& "&") (= "=") @@ -81,6 +88,7 @@ "format" = (symbol->string format) & "language" = (symbol->string language)))) (string->uri + ;; Handle optional arguments. (if (symbol? property) (let ((property (symbol->string property))) (string-append u & "props" = property)) @@ -101,15 +109,15 @@ (string->uri u))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -;; Medium-level -;; Extract from queries +;; Medium-level proc. +;; Extract data from queries ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; (define (get-label qid) (let ((l "label")) (first (extract-all - (wdquery (getentities-uri qid l)))) + (wdquery-alist (getentities-uri qid l)))) )) (define (get-properties qid) @@ -117,7 +125,7 @@ (map car ;show all P only first P-statement (cdr ; ->((p1)(p2(...)))) = list of properties (first ; ->(claims ((p1)(p2(...)))) - (wdquery + (wdquery-alist (getclaims-uri qid)))))) ;;test @@ -172,26 +180,37 @@ (x 50)) (if (> (string-length ld) x) (string-append (substring ld 0 x) "...") - ld - )))) + ld)))) -(define (extract-search name) +(define* (extract-search name + #:key + (language 'en) + (limit 'limit)) "Returns list with each element being an alist of label, desc, qid" - (map extract-all (assoc-ref (wdquery (search-uri name)) "search"))) + (map extract-all + (assoc-ref + (wdquery-alist + (search-uri name + #:language language + #:limit limit)) "search"))) ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; High-level -;; Get the results fast +;; Get search results fast ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(define (search query x) - "Extract first x results and print them in a pretty truncated way." - (let ((result (extract-search query))) +(define* (search query + #:key + (language 'en) + (limit 10)) ;default to 10 + "Extract results until 'limit' and print them in a pretty truncated way." + (let ((result (extract-search query + #:language language + #:limit limit))) (begin - (format #t "First ~a:\tLabel & Description~%" x) - (map pretty-print (take result x)) - ) - )) + (format #t "First ~a:\tLabel & Description~%" limit) + (map pretty-print (take result limit))))) ;; For example: -;; (search "aragorn" 10) +(search "paris" #:limit 15 #:language 'es) + diff --git a/wikidata/apis.scm~ b/wikidata/apis.scm~ new file mode 100644 index 0000000..3bea6f5 --- /dev/null +++ b/wikidata/apis.scm~ @@ -0,0 +1,290 @@ +;;; Copyright © 2018 swedebugia <swedebugia@riseup.net> +;;; +;;; This file is part of guile-wikidata. +;;; +;;; guile-wikidata is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; guile-wikidata is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with guile-wikidata. If not, see <http://www.gnu.org/licenses/>. + +;;; See +;;; https://www.mediawiki.org/wiki/API:Presenting_Wikidata_knowledge +;;; for a good description of workflow when integrating the Wikidata API into +;;; an application. + +;;; To generate SPARQL queries I recommend +;;; https://query.wikidata.org/. + +(define-module (wikidata) + #:use-module (ice-9 format) + #:use-module (ice-9 optargs) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) + #:use-module (json) + #:use-module (guix http-client) + #:use-module (guix import utils) +;; #:use-module (sparql driver) ; does not support blazegraph +;; #:use-module (sparql lang) ; did not work ?? + #:use-module (sparql util) + #:use-module (guix import json) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-34) + #:use-module (sxml simple) + #:use-module (web uri) + #:export (show)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Low-level proc. +;; URI-decorators and fetching +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; TODO flesh out http-code from giux into a new guile-http library to +;; avoid pulling in all of guix in this library. +(define* (xml-fetch url + ;; Note: many websites returns 403 if we omit a + ;; 'User-Agent' header. + #:key (headers `((user-agent . "GNU Guile") + (Accept . "application/json")))) + "Return a representation of the JSON resource URL (a list or hash table), or +#f if URL returns 403 or 404. HEADERS is a list of HTTP headers to pass in +the query. Returns RESULT from port." + (guard (c ((and (http-get-error? c) + (let ((error (http-get-error-code c))) + (or (= 403 error) + (= 404 error)))) + #f)) + (let* ((port (http-fetch url #:headers headers)) + ;; Return result without any modification. + (result port)) +;; (close-port port) + result))) + +;;TODO implement caching +(define (wdquery-alist uri) + "Fetch the data, return an alist" + (json-fetch-alist uri)) + +(define (wdquery-xml uri) + "Fetch the data, return PORT" + (xml-fetch uri)) + +;; Inspired by PYPI wikidata_suggest +(define* (search-uri name + #:key + (format 'json) + (language 'en) + (type 'item) + (continue 0) + (limit 10)) + + "Build URI for the Wikidata wbsearchentities API." + (let ((url "https://www.wikidata.org/w/api.php") + (& "&") + (= "=")) + (string->uri + (string-append url "?" + "search" = name & + "action" = "wbsearchentities" & + "format" = (symbol->string format) & + "language" = (symbol->string language) & + "type" = (symbol->string type) & + "continue" = (number->string continue) & + "limit" = (number->string limit) + )))) + +;; Inspired by +;; https://opendata.stackexchange.com/questions/5248/how-to-get-the-name-of-a-wikidata-item +;; TODO add handling of more than one qid. +(define* (getentities-uri qid + #:optional property + #:key (language 'en) + (format 'json)) + "Build URI for the Wikidata wbsearchintities API. PROPERTY is a +string containing one of the props here: +https://www.wikidata.org/wiki/Special:ApiHelp/wbgetentities" + (let* ((url "https://www.wikidata.org/w/api.php") + (& "&") + (= "=") + (u (string-append url "?" + "ids" = qid & + "action" = "wbgetentities" & + "format" = (symbol->string format) & + "language" = (symbol->string language)))) + (string->uri + ;; Handle optional arguments. + (if (symbol? property) + (let ((property (symbol->string property))) + (string-append u & "props" = property)) + ;; No property + u)))) + +;; Only one at a time. +(define* (getclaims-uri qid + #:key (format 'json)) + "Build URI for the Wikidata wbgetclaims API." + (let* ((url "https://www.wikidata.org/w/api.php") + (& "&") + (= "=") + (u (string-append url "?" + "entity" = qid & + "action" = "wbgetclaims" & + "format" = (symbol->string format)))) + (string->uri u))) + +;; Inspired by http://r.duckduckgo.com/l/?kh=-1&uddg=http%3A%2F%2Fstackoverflow.com%2Fquestions%2F29886388%2Fddg%2335118127 +;;; +;;; Wikidata-specific SPARQL-QUERY using a GET request (it also accept POST) +;;; --------------------------------------------------------------------------- +(define* (wdsparql-uri query + #:key + (uri #f) + (type "json")) + "Build URI for the Wikidata HTTP GET SPARQL API." + (let* ((get-uri "http://query.wikidata.org/sparql") + (get-url (if uri + uri + get-uri))) + (string->uri + (string-append get-url "?" (uri-encode query)) + ) + )) + +;; PREFIX wd: <http://www.wikidata.org/entity/> +;; PREFIX wdt: <http://www.wikidata.org/prop/direct/> + +;; SELECT DISTINCT ?item +;; WHERE { +;; ?item wdt:P31/wdt:P279* wd:Q19723451 +;; } + +;; broken +;; (display-query-results-of +;; (wdquery-xml +;; (wdsparql-uri +;; "PREFIX wd: <http://www.wikidata.org/entity/> +;; PREFIX wdt: <http://www.wikidata.org/prop/direct/> SELECT DISTINCT +;; ?item WHERE \\{?item wdt:P31/wdt:P279* wd:Q19723451\\}" +;; ))) + +;; Broken attempt to use SEXP from (sparql lang). +;; (let ( +;; (wd (prefix "http://www.wikidata.org/entity/")) +;; (wdt (prefix "http://www.wikidata.org/prop/direct/"))) +;; (select #:destinct +;; ;; columns +;; ;;'(subject predicate object) +;; `((,'item))) +;; ;; pattern +;; ;; `((subject predicate object) +;; ;; (subject ,(rdf "type") ,(internal "Sample") +;; `(((,'where ,'item) (,wdt ":P31/" ,wdt ":P279*") (,wd ":Q19723451"))) +;; ))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Medium-level proc. +;; Extract data from queries +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(define (get-label qid) + (let ((l "label")) + (first + (extract-all + (wdquery-alist (getentities-uri qid l)))) + )) + +(define (get-properties qid) + "Fetches and RETURNS list of all P-property keys from a qid" + (map car ;show all P only first P-statement + (cdr ; ->((p1)(p2(...)))) = list of properties + (first ; ->(claims ((p1)(p2(...)))) + (wdquery-alist + (getclaims-uri qid)))))) + +;;test +;;(display (map getlabel (get-properties "Q180736"))) + +;; TODO factorize nested alist check +(define* (extract-element alist element + #:optional (x 30)) + "Accept unnested ALIST and return the value of ELEMENT. +'qid. Truncate elements to X char, default to 30 char." + (if (if (member element '("label" "description" "id")) #t #f) + ;; True + (if (list? alist) + (let ((result (assoc-ref alist element))) + (if (string? result) + ;; Truncate string + (if (> (string-length result) x) + (string-append (substring result 0 x) "...") + result) + (if (null? result) + (begin + (error "extract-element: No" element "found:") + (display alist)) + (string-append + "(No " element " in the database)")))) + (begin + (error "extract-element: Not a proper list:" ) + (display alist))) + ;; Not one of the accepted elements + (error "extract-element: accepts only the strings: label, description or id"))) + +(define (extract-all alist) + "Extract all elements for a given unnested alist" + (if (list? alist) + (if (not (= 0 (length alist))) + `(("label" . ,(extract-element alist "label")) + ("description" . ,(extract-element alist "description")) + ("id" . ,(extract-element alist "id"))) + (error "extract-all: Nothing found." )) + (begin + (error "extract-all: Not a proper list:") + (display alist)))) + +(define (pretty-print result) + "Takes an unnested alist RESULT and pretty prints it." + (format #t "~a:\t~a~%" + (extract-element result "id") + ;; Join and truncate long labels and descriptions + (let* ((l (extract-element result "label")) + (d (extract-element result "description")) + (ld (string-append l ": " d)) + (x 50)) + (if (> (string-length ld) x) + (string-append (substring ld 0 x) "...") + ld + )))) + +(define* (extract-search name + #:key language) + "Returns list with each element being an alist of label, desc, qid" + (map extract-all + (assoc-ref (wdquery-alist + (search-uri name #:language language)) "search"))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; High-level +;; Get the results fast +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +(define* (search query x + #:key language) + "Extract first x results and print them in a pretty truncated way." + (let ((result (extract-search query))) + (begin + (format #t "First ~a:\tLabel & Description~%" x) + (map pretty-print (take result x)) + ) + )) + +;; For example: +;;(search "guix" 10 #:language 'fr) + diff --git a/wikidata/sparql.scm b/wikidata/sparql.scm new file mode 100644 index 0000000..5b71152 --- /dev/null +++ b/wikidata/sparql.scm @@ -0,0 +1,111 @@ +;;; Copyright © 2018 swedebugia <swedebugia@riseup.net> +;;; +;;; This file is part of guile-wikidata. +;;; +;;; guile-wikidata is free software; you can redistribute it and/or modify it +;;; under the terms of the GNU General Public License as published by +;;; the Free Software Foundation; either version 3 of the License, or (at +;;; your option) any later version. +;;; +;;; guile-wikidata is distributed in the hope that it will be useful, but +;;; WITHOUT ANY WARRANTY; without even the implied warranty of +;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;;; GNU General Public License for more details. +;;; +;;; You should have received a copy of the GNU General Public License +;;; along with guile-wikidata. If not, see <http://www.gnu.org/licenses/>. + +;; See the example-sparql.scm for how to use this library. + +(define-module (wikidata sparql) + #:use-module (ice-9 format) + #:use-module (ice-9 rdelim) + #:use-module (ice-9 receive) + #:use-module (guix http-client) +;; #:use-module (guix import utils) ; useful stuff there +;; #:use-module (sparql driver) ; does not support blazegraph +;; #:use-module (sparql lang) ; did not work ?? + #:use-module (sparql util) + #:use-module (srfi srfi-1) + #:use-module (srfi srfi-34) + #:use-module (web uri) + #:export (show-sparql)) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Low-level proc. +;; URI-decorators and fetching +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + +;; TODO flesh out http-code from giux into a new guile-http library to +;; avoid pulling in all of guix in this library. +(define* (xml-fetch url + ;; Note: many websites returns 403 if we omit a + ;; 'User-Agent' header. + #:key (headers `((user-agent . "GNU Guile") + (Accept . "application/json")))) + "Return a representation of the JSON resource URL (a list or hash table), or +#f if URL returns 403 or 404. HEADERS is a list of HTTP headers to pass in +the query. Returns RESULT from port." + (guard (c ((and (http-get-error? c) + (let ((error (http-get-error-code c))) + (or (= 403 error) + (= 404 error)))) + #f)) + (let* ((port (http-fetch url #:headers headers)) + ;; Return result without any modification. + (result port)) +;; (close-port port) + result))) + +(define (wdquery-xml uri) + "Fetch the data, return PORT" + (xml-fetch uri)) + + +;; Inspired by http://r.duckduckgo.com/l/?kh=-1&uddg=http%3A%2F%2Fstackoverflow.com%2Fquestions%2F29886388%2Fddg%2335118127 +;;; +;;; Wikidata-specific SPARQL-QUERY using a GET request (it also accept POST) +;;; --------------------------------------------------------------------------- +(define* (wdsparql-uri query + #:key + (uri #f) + (type "json")) + "Build URI for the Wikidata HTTP GET SPARQL API." + (let* ((get-uri "http://query.wikidata.org/sparql") + (get-url (if uri + uri + get-uri))) + (string->uri + (string-append get-url "?" (uri-encode query)) + ) + )) + + +;;; +;;; Medium-level proc. +;;; --------------------------------------------------------------------------- + +;; TODO add rationale for why this is copied from (sparql util) +(define (display-query-results port) + "Format the output from the port and close it" + (begin + (let ((line (read-line port))) + (if (eof-object? line) + #t + ;; The default output format is comma-separated values (CSV). + (let ((tokens (string-split line #\,))) + (format #t "~{~a~/~}~%" tokens) + (display-query-results port)))) + (close-port port))) + +;;; +;;; High-level proc. +;;; --------------------------------------------------------------------------- + +;; See example-sparql.scm for how to enter the query +(define (show-sparql query) + "Run the query on the Wikidata Blazegraph server. Show the result on current-output-port." + (display-query-results + (xml-fetch + (wdsparql-uri + query)))) |