wiki.erg.be/map
1
0
Fork 0
Code Issues 4 Pull Requests Packages Projects Releases Wiki Activity

merge forcenet with wikimap, many refinements, incremental loading directly from api

Browse Source
This commit is contained in:
Michael Murtaugh
2019-07-09 17:31:52 +02:00
parent ad3e494d47
commit 5ac253c712
6 changed files with 1276 additions and 966 deletions
Download Patch File Download Diff File Expand all files Collapse all files

598
src/wikimap.js
View File

@@ -7,153 +7,72 @@ import { Wiki, Page } from './wiki.js';
import EventEmitter from 'eventemitter3';
import { json } from 'd3-fetch';
import { ForceNet } from './forcenet.js';
// import { ForceNet } from './forcenet.js';
export class Map {
constructor (symbols) {
constructor (opts) {
var width = 600,
height = 600;
this.apiurl = opts.apiurl;
this.init_svg(opts.svg);
this.categorylabel = opts.categorylabel || "Category";
this.symbols_src = opts.symbols;
this.categorydiv = select(opts.categorydiv);
this.wiki = new Wiki(this.apiurl);
this.events = new EventEmitter();
this.active_page = null;
// this.nodes = {};
this.symbols = symbols;
this.net = new ForceNet(symbols);
this.net.on("nodeclick", this.nodeclick.bind(this));
// this.simulation = forceSimulation()
// .velocityDecay(0.1)
// .force("link", forceLink().id(d => d.title))
// .force("charge", forceManyBody())
// .force("radial", forceRadial(180, width/2, height/2));
// // .force("center", forceCenter(width / 2, height / 2));
// this.symbols = symbols;
// this.net = new ForceNet({});
// this.net.on("nodeclick", this.nodeclick.bind(this));
this.svg = null;
this.historylinks = {};
this.links = null;
this.highlight_category = null;
this.show_history = true;
this.loaded = false;
this.active_url = null;
this.active_page = null;
this.simulation = forceSimulation()
//.velocityDecay(0.1)
.force("link", forceLink().id(d => d.title))
.force("charge", forceManyBody())
.force("radial", forceRadial(180, width/2, height/2));
// .force("center", forceCenter(width / 2, height / 2));
this.all_links_by_key = {};
}
nodeclick (d, elt) {
console.log("nodeclick", d, elt, this);
this.set_active_node(d, elt);
}
async init () {
this.symbols = await json(this.symbols_src);
await this.wiki.init();
init_svg (svg) {
this.net.init_svg(svg);
}
async load_json (source) {
var data = await json(source);
// console.log("got data!", data);
// index the nodes by title, init link-arity count
var index = {};
this.nodes_by_title = index;
for (let i=0, l=data.nodes.length; i<l; i++) {
let node = data.nodes[i];
node.count = 0;
index[node.title] = node;
}
this.nodes = data.nodes;
var use_links = [];
data.links = data.links.forEach(x => {
var source = index[x.source],
target = index[x.target];
if (source === undefined) {
console.log("bad source", x.source);
return;
}
if (target === undefined) {
console.log("bad target", x.target);
return;
}
source.count += 1;
target.count += 1;
use_links.push({ source: source, target: target });
});
data.links = use_links;
this.links = data.links;
// console.log("data", data);
// calculate the node sizes (link arity)
this.net.update_graph(data);
}
get_symbol_image_path (cname) {
var symbol = this.symbols[cname];
if (symbol) {
let hpos = symbol.indexOf("#"),
rest = symbol.substr(hpos+1);
rest = rest.replace(/'/g, '');
return "img/"+rest+".png";
}
}
async load_cats (src, elt) {
var data = await json(src);
console.log("indexing categories by title");
var cats_by_title = {};
for (let i=0, l=data.length; i<l; i++) {
let cat = data[i];
cats_by_title[cat.title] = cat;
cat.pages = [];
cat.tcount = 0;
}
// index categories
console.log("indexing categories");
for (let key in this.nodes_by_title) {
let node = this.nodes_by_title[key];
// console.log("key", key, node.cats);
for (let j=0, jl=node.cats.length; j<jl; j++) {
let cname = node.cats[j],
cat = cats_by_title[cname];
if (cat) {
// increment the category + parents
cat.pages.push(node);
cat.tcount += 1
while (cat.parent) {
cat = cats_by_title[cat.parent];
cat.tcount += 1;
}
} else {
console.log("Warning, unknown category", cname);
// load categories & set their page symbols
for (let i=0, l=this.symbols.length; i<l; i++) {
let sym = this.symbols[i];
if (sym.key === "default") {
let page = this.wiki.get_page_by_title("Special:AllPages");
sym.page = page;
} else {
let cat = this.wiki.get_page_by_title(this.categorylabel+":"+sym.key);
sym.page = cat;
cat.pages = await cat.get_categorymembers();
// console.log("got cat pages", cat);
for (let j=0, jlen=cat.pages.length; j<jlen; j++) {
let cp = cat.pages[j];
cp.symbol = sym.symbol;
}
}
}
console.log("pre filter", data.length);
data = data.filter(d => d.tcount > 0)
console.log("post filter", data.length);
console.log("load_cats.data", data, elt);
var cat = select(elt)
.selectAll("div.cat")
.data(data)
.enter()
.append("div")
.attr("class", "cat");
cat.classed("icon", d => this.get_symbol_image_path(d.title));
cat.append("span").attr("class", "icon").filter(d=> this.get_symbol_image_path(d.title)).style("background-image", d => "url("+this.get_symbol_image_path(d.title)+")");
cat.append("span").attr("class", "spacing").html(d => {
var d = d.depth,
ret = "";
while(d) {
ret += "&nbsp;&nbsp;&nbsp;&nbsp;";
d-=1;
}
return ret;
});
cat.append("a").attr("class", "label").html(d => d.title).attr("href", "#").on("click", d => {
event.preventDefault();
this.category_click(d);
})
cat.append("span").attr("class", "count").html(d => d.tcount)
}
async load_legend (src, elt) {
var data = await json(src);
console.log("load_legend", data);
var cat = select(elt)
// create the categories
let cat = this.categorydiv
.selectAll("div.cat")
.data(data)
.data(this.symbols)
.enter()
.append("div")
.attr("class", "cat icon");
@@ -162,94 +81,233 @@ export class Map {
.style("background-image", d => "url("+d.icon+")");
cat.append("a")
.attr("class", "label")
.html(d => d.key)
.html(d => d.key === "default" ? "Page" : d.key)
.attr("href", "#").on("click", d => {
event.preventDefault();
this.category_click(d);
})
cat.append("span").attr("class", "count").html(d => d.tcount)
}
category_click (d) {
console.log("category click", d);
if (this.highlight_category) {
// cleanup old pages
this.highlight_category.pages.forEach(d => d.highlight = false);
this.loaded = true;
if (this.active_url) {
this.set_active_url(this.active_url);
}
this.highlight_category = d;
this.highlight_category.pages.forEach(d => d.highlight = true);
this.net.update_nodes();
// set highlight category...
// all nodes with this category get .highlight = true
// make a category index ?!
}
on (message, callback, context) {
this.events.on(message, callback, context);
}
/* OLD STYLE with node.all_links
walk (node, links) {
var links_seen = {};
// var node = this.ensure_node(nodename);
node.all_links.forEach(x => {
var link_key = this.link_key(node.title, x.title);
if (!links_seen[link_key]) {
links.push({source: node, target: x});
links_seen[link_key] = true;
}
})
return;
}
*/
set_active_title (title) {
this.set_active_node(title);
async json (src) {
// expose d3's json method
return await json(src);
}
activate_linked_nodes (page, active) {
// deactivate linked links/nodes
for (let i=0, l=this.links.length; i<l; i++) {
let link = this.links[i];
if (link.source == page || link.target == page) {
link.active2 = active;
link.source.active2 = active;
link.target.active2 = active;
}
}
init_svg (svg) {
this.svg = select(svg || "svg");
this.zoom = zoom()
.scaleExtent([1 / 16, 16])
.on("zoom", () => {
this.content.attr("transform", event.transform);
// console.log("transform", event.transform, this.content.attr("transform"));
});
this.rect = this.svg.append("rect")
.attr("width", 1000)
.attr("height", 1000)
.style("fill", "none")
.style("pointer-events", "all")
.call(this.zoom);
this.content = this.svg.append("g")
.attr("id", "content"),
this.linksg = this.content.append("g")
.attr("class", "links");
this.nodesg = this.content.append("g")
.attr("class", "nodes");
}
async set_active_node (page) {
if (typeof(page) === "string") {
let pagename = page;
page = this.nodes_by_title[page];
dragstarted (d) {
// if (!event.active) this.simulation.alphaTarget(0.3).restart();
d.fx = d.x;
d.fy = d.y;
}
if (!page) {
console.log("wikimap.set_active_node: page not found", pagename);
}
dragged (d) {
d.fx = event.x;
d.fy = event.y;
}
dragended(d) {
// if (!event.active) this.simulation.alphaTarget(0);
d.fx = null;
d.fy = null;
}
/* Links */
link_key (p1, p2) {
return (p1.title < p2.title) ?
("link_"+p1.title+"_"+p2.title) :
("link_"+p2.title+"_"+p1.title);
}
make_link (p1, p2) {
return (p1.title < p2.title) ?
{source: p1, target: p2 } :
{source: p2, target: p1 };
}
ensure_link (from_page, to_page) {
var lkey = this.link_key(from_page, to_page),
ret = this.all_links_by_key[lkey];
if (ret === undefined) {
ret = this.make_link(from_page, to_page);
this.all_links_by_key[lkey] = ret;
}
return ret;
}
async set_active_url (url) {
this.active_url = url;
if (this.loaded) {
var page = this.wiki.get_page(this.active_url);
if (page) {
this.set_active_page(page);
} else {
console.log("wikimap: set_active_url: NO PAGE FOR", url);
}
}
}
linked_nodes_set_active (page, active) {
// deactivate linked links/nodes
for (var key in this.all_links_by_key) {
if (this.all_links_by_key.hasOwnProperty(key)) {
var link = this.all_links_by_key[key];
if (link.source == page || link.target == page) {
link.active2 = active;
link.source.active2 = active;
link.target.active2 = active;
}
}
}
}
clear_highlight_category () {
if (this.highlight_category) {
// cleanup old pages
this.highlight_category.highlight = false;
this.highlight_category.pages.forEach(d => d.highlight = false);
}
if (this.active_page == this.highlight_category) {
this.active_page = null;
}
this.highlight_category = null;
this.categorydiv
.selectAll("div.cat")
.classed("highlight", d=> d.page ? d.page.highlight : false);
var data = {nodes: this.wiki.get_nodes(), links: values(this.all_links_by_key)};
this.update_node_counts()
this.update_graph(data);
}
async set_active_page (page) {
console.log("wikimap: set_active_page:", page.title);
if (page === this.active_page) {
// console.log("page is already the active page", page, this.active_page);
return;
}
// cleanup old
if (this.active_page) {
this.active_page.active = false;
// deactivate linked links/nodes
this.activate_linked_nodes(this.active_page, false);
// ENSURE HISTORY LINK TO PREVIOUS NODE AND CURRENT
var lkey = this.net.link_key(this.active_page.title, page.title),
source = (this.active_page.title < page.title) ? this.active_page : page,
target = (this.active_page.title < page.title) ? page : this.active_page;
this.historylinks[lkey] = {source: source, target: target, type:"history"};
this.linked_nodes_set_active(this.active_page, false);
// ENSURE (HISTORY) LINK TO PREVIOUS NODE AND CURRENT
//var link = this.ensure_link(this.active_page, page);
// link.visited = true;
}
if (page.ns == 14) {
this.active_page = page;
if (!page.pages) {
page.pages = await page.get_categorymembers();
}
if (this.highlight_category) {
// cleanup old pages
this.highlight_category.highlight = false;
this.highlight_category.pages.forEach(d => d.highlight = false);
}
this.highlight_category = page;
this.highlight_category.highlight = true;
this.highlight_category.pages.forEach(d => d.highlight = true);
// this.update_nodes();
this.categorydiv
.selectAll("div.cat")
.classed("highlight", d=> d.page ? d.page.highlight : false);
// repetition of below... (could be improved)
// this.events.emit("page", this.active_page);
// var data = {nodes: this.wiki.get_nodes(), links: values(this.all_links_by_key)};
// this.update_node_counts()
// this.update_graph(data);
// return;
} else if (page.ns !== 0) {
console.log("SPECIAL PAGE", page);
} else {
// LOAD/ENSURE PAGE LINKS
var links_out = await page.get_links();
links_out = links_out.filter(p => (!p.redirect && p.ns == 0));
console.log("links_out", links_out);
var links_in = await page.get_linkshere();
links_in = links_in.filter(p => (!p.redirect && p.ns == 0));
console.log("links_in", links_in);
links_out.forEach(p => {
this.ensure_link(page, p).wiki = true;
});
links_in.forEach(p => {
this.ensure_link(p, page).wiki = true;
});
}
this.active_page = page;
this.active_page.active = true;
this.activate_linked_nodes(this.active_page, true);
this.linked_nodes_set_active(this.active_page, true);
this.events.emit("page", this.active_page.title);
this.events.emit("page", this.active_page);
this.net.update_nodes();
this.net.update_forces();
var data = {nodes: this.wiki.get_nodes(), links: values(this.all_links_by_key)};
this.update_node_counts()
this.update_graph(data);
// this.update_nodes();
// this.update_forces();
}
update_node_counts () {
var nodes = this.wiki.get_nodes();
for (let i=0, len=nodes.length; i<len; i++) {
nodes[i].count = 0;
}
for (var key in this.all_links_by_key) {
if (this.all_links_by_key.hasOwnProperty(key)) {
var link = this.all_links_by_key[key];
link.source.count += 1;
link.target.count += 1;
}
}
}
category_click (d) {
console.log("category click", d);
if (d.page) {
if (this.highlight_category == d.page) {
this.clear_highlight_category();
} else {
this.set_active_page(d.page);
}
}
}
set_show_history (value) {
@@ -263,44 +321,160 @@ export class Map {
for (var key in this.historylinks) {
graph.links.push(this.historylinks[key])
}
this.net.update_graph(graph);
this.update_graph(graph);
} else {
let graph = {};
graph.nodes = this.nodes;
graph.links = this.links;
this.net.update_graph(graph);
this.update_graph(graph);
}
}
}
/*
async load (page) {
console.log("loading", page.title);
var links = await page.get_links(),
backlinks = await page.get_linkshere(),
alllinks = this.wiki.union(links, backlinks);
page.all_links = alllinks;
// console.log("alllinks", alllinks);
var graph = {};
update_graph (graph) {
console.log("UPDATE GRAPH", graph.nodes.length, graph.links.length);
var link = this.linksg.selectAll("line")
.data(graph.links, d => { return this.link_key(d.source.title, d.target.title) });
var link_enter = link.enter()
.append("line");
graph.nodes = values(this.wiki.pages_by_title);
graph.links = [];
this.walk(page, graph.links);
// activate historylinks
values(this.historylinks).forEach(x => {
graph.links.push(x);
})
// graph.links = titles.map(t => ({source: pagetitle, target: t}));
this.update_graph(graph);
// console.log("GOT DATA", titles);
// return titles;
link.exit().each(d => {
d.source.linked = false;
d.target.linked = false;
}).remove();
link_enter.merge(link).each(d => {
d.source.linked = true;
d.target.linked = true;
});
var node = this.nodesg
.selectAll("g.page")
.data(graph.nodes, function (d) { return d.title });
node.exit().remove();
var that = this;
var node_enter = node.enter().append("g")
.attr("class", "page")
// .attr("class", d=>"page "+this.wiki.get_ns_classname(d.ns))
.on("click", d => {
// that.events.emit("nodeclick", d, this);
this.set_active_page(d);
// this.set_active_node(d.title);
})
.on("mouseover", function (d) {
// console.log("mouseover", this);
select(this).classed("mouse", true);
})
.on("mouseout", function (d) {
// console.log("mouseout", this);
select(this).classed("mouse", false);
})
.call(drag()
.on("start", this.dragstarted.bind(this))
.on("drag", this.dragged.bind(this))
.on("end", this.dragended.bind(this)));
node_enter.append("use")
.attr("xlink:href", d => d.symbol || "symbols.svg#Main")
.attr("class", "testcolor");
// {
// for (var i=0, l=d.cats.length; i<l; i++) {
// if (this.symbols[d.cats[i]]) {
// return this.symbols[d.cats[i]];
// }
// }
// return this.symbols.default || "default";
// });
// node_enter.append("circle")
// .attr("r", 6);
node_enter.append("text")
.text(d => d.title)
.attr("x", 10);
//node_enter.append("title")
// .text(function(d) { return d.title; });
node = node_enter.merge(node);
link = link_enter.merge(link);
node.classed("active", d=>d.active);
this.simulation
.nodes(graph.nodes)
.on("tick", ticked);
this.simulation.force("link")
.links(graph.links);
this.simulation.force("radial").radius(d => (d.linked || d.highlight) ? null : 200);
function ticked() {
link
.attr("x1", function(d) { return d.source.x; })
.attr("y1", function(d) { return d.source.y; })
.attr("x2", function(d) { return d.target.x; })
.attr("y2", function(d) { return d.target.y; });
// node
// .attr("cx", function(d) { return d.x; })
// .attr("cy", function(d) { return d.y; });
node
.attr("transform", d => `translate(${d.x},${d.y})`);
}
// document.querySelector("#page").style.background = "purple";
// return;
this.update_nodes();
this.update_forces();
// this.simulation.alphaTarget(0.3).restart();
}
*/
update_nodes () {
var nodes = this.nodesg.selectAll("g.page");
// console.log("update_nodes", nodes.size());
nodes.classed("active", d=>d.active);
nodes.classed("active2", d=>d.active2);
nodes.classed("highlight", d=>d.highlight);
nodes.sort((a, b) => {
// console.log("sort", a, b);
var x = a.active ? 10 : (a.active2 ? 5 : 0),
y = b.active ? 10 : (b.active2 ? 5 : 0);
return x - y;
});
var links = this.linksg.selectAll("line");
links.classed("active2", d=>d.active2);
links.classed("history", d=>d.type == "history");
links.sort((a, b) => {
// console.log("sort", a, b);
var x = a.active2 ? 10 : (a.history ? 5 : 0),
y = b.active2 ? 10 : (b.history ? 5 : 0);
return x - y;
});
}
update_forces () {
var force = this.simulation.force("link");
// console.log("update_forces:force", force);
this.simulation.force("link").strength(d => {
if (d.source.active || d.target.active) {
return 1;
} else {
// same as d3.force's defaultStrength
return 0.5 * (1 / Math.min(d.source.count, d.target.count));
}
});
// this.simulation.alphaTarget(0.3).restart();
this.simulation.alpha(0.5).restart();
}
}
// http://erg.activearchives.org/mw/api.php?action=query&prop=links&titles=Bienvenue_%C3%A0_l%E2%80%99erg
// http://erg.activearchives.org/w/api.php?action=query&prop=info&titles=Main%20Page
// Bienvenue_à_l’erg
// http://erg.activearchives.org/mw/index.php/Bienvenue_%C3%A0_l%E2%80%99erg